You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by is...@apache.org on 2025/05/08 08:37:38 UTC
(ignite-3) branch main updated: IGNITE-24234 Implement Migration Tools for data migration from Ignite 2 to Ignite 3 (#5605)
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new e8397c56638 IGNITE-24234 Implement Migration Tools for data migration from Ignite 2 to Ignite 3 (#5605)
e8397c56638 is described below
commit e8397c56638e722489ea37b235ecc0477fe4ec93
Author: Tiago Marques Godinho <me...@gmail.com>
AuthorDate: Thu May 8 09:37:34 2025 +0100
IGNITE-24234 Implement Migration Tools for data migration from Ignite 2 to Ignite 3 (#5605)
---
build.gradle | 21 +
gradle.properties | 3 +
gradle/libs.versions.toml | 18 +-
migration-tools/.gitignore | 26 +
migration-tools/README.md | 198 +++++
migration-tools/log4j2-test.xml | 30 +
migration-tools/modules/e2e-ai3-tests/build.gradle | 41 +
.../e2e/DataMigrationWithIgnite3ClientTest.java | 156 ++++
.../modules/ignite2-repack/build.gradle | 54 ++
.../modules/ignite3-repack/build.gradle | 110 +++
.../modules/ignite3-slim-repack/build.gradle | 83 ++
.../migration-tools-ai2-test-fixtures/build.gradle | 29 +
.../org/apache/ignite/examples/model/Address.java | 72 ++
.../apache/ignite/examples/model/Organization.java | 132 ++++
.../ignite/examples/model/OrganizationType.java | 32 +
.../org/apache/ignite/examples/model/Person.java | 148 ++++
.../apache/ignite/examples/model/package-info.java | 23 +
.../modules/migration-tools-cli/build.gradle | 45 ++
.../org/apache/ignite/migrationtools/cli/Main.java | 125 +++
.../migrationtools/cli/TopLevelCliCommand.java | 51 ++
.../configs/calls/ConfigurationConverterCall.java | 102 +++
.../commands/ConfigurationConverterCmd.java | 59 ++
.../IgniteClientConnectionExceptionHandler.java | 37 +
.../cli/mixins/ClassloaderOption.java | 76 ++
.../cli/persistence/calls/ListCachesCall.java | 111 +++
.../cli/persistence/calls/MigrateCacheCall.java | 430 +++++++++++
.../calls/RetriableMigrateCacheCall.java | 137 ++++
.../cli/persistence/commands/ListCachesCmd.java | 45 ++
.../cli/persistence/commands/MigrateCacheCmd.java | 68 ++
.../persistence/commands/PersistenceBaseCmd.java | 97 +++
.../cli/persistence/params/MigrateCacheParams.java | 98 +++
.../cli/persistence/params/MigrationMode.java | 39 +
.../cli/persistence/params/PersistenceParams.java | 43 ++
.../params/RetrieableMigrateCacheParams.java | 49 ++
.../cli/sql/calls/SqlDdlGeneratorCall.java | 144 ++++
.../cli/sql/commands/SqlDdlGeneratorCmd.java | 67 ++
.../src/main/resources/log4j2.xml | 30 +
.../calls/RetriableMigrateCacheCallTest.java | 161 ++++
.../migration-tools-commons-tests/build.gradle | 63 ++
.../SkipUnsupportedOperationsHandlers.java | 51 ++
.../tests/bases/MigrationTestBase.java | 152 ++++
.../tests/clusters/FullSampleCluster.java | 51 ++
.../tests/containers/Ignite2ClusterContainer.java | 233 ++++++
.../containers/Ignite2ClusterWithSamples.java | 142 ++++
.../tests/containers/Ignite3ClusterContainer.java | 186 +++++
.../tests/models/ComplexKeyIntStr.java | 58 ++
.../migrationtools/tests/models/SimplePojo.java | 82 ++
.../tests/utils/RunIgnite2Cluster.java | 43 ++
.../tests/utils/RunIgnite3Cluster.java | 37 +
.../modules/migration-tools-commons/build.gradle | 37 +
.../config/Ignite2ConfigurationUtils.java | 99 +++
.../config/Ignite3ConfigurationUtils.java | 142 ++++
.../config/loader/BeanWrapperSpy.java | 249 ++++++
.../config/loader/CustomBeanFactory.java | 93 +++
.../config/registry/CombinedConfigRegistry.java | 60 ++
.../registry/ConfigurationRegistryInterface.java | 29 +
.../storage/NoDefaultsStorageConfiguration.java | 72 ++
.../sql/FieldNameConflictException.java | 54 ++
.../ignite/migrationtools/sql/SqlDdlGenerator.java | 841 +++++++++++++++++++++
.../migrationtools/tablemanagement/Namespace.java | 51 ++
.../PersistentTableTypeRegistryImpl.java | 94 +++
.../RegisterOnlyTableTypeRegistry.java | 41 +
.../tablemanagement/SchemaUtils.java | 51 ++
.../tablemanagement/TableTypeRegistry.java | 44 ++
.../tablemanagement/TableTypeRegistryMapImpl.java | 46 ++
.../converters/CalendarToInstantConverter.java | 38 +
.../CalendarToLocalDateTimeConverter.java | 38 +
.../types/converters/DateToInstantConverter.java | 35 +
.../converters/DateToLocalDateTimeConverter.java | 36 +
.../converters/EnumToStringTypeConverter.java | 39 +
.../types/converters/IdentityConverter.java | 35 +
.../converters/SqlDateToLocalDateConverter.java | 35 +
.../converters/SqlTimeToLocalTimeConverter.java | 35 +
.../converters/SqlTimestampToInstantConverter.java | 35 +
.../SqlTimestampToLocalDateTimeConverter.java | 35 +
.../converters/StaticTypeConverterFactory.java | 70 ++
.../types/converters/TypeConverterFactory.java | 27 +
.../migrationtools/utils/ClassnameUtils.java | 44 ++
.../internal/catalog/sql/CatalogExtensions.java | 31 +
.../sql/sql/SqlDdlGeneratorTest.java | 393 ++++++++++
.../migration-tools-config-converter/build.gradle | 56 ++
.../config/ClusterLoadsConfigTest.java | 56 ++
.../config/ConfigurationConverter.java | 125 +++
.../converters/ClientConnectorConverter.java | 53 ++
.../converters/CommunicationSpiConverter.java | 57 ++
.../config/converters/ConfigurationConverter.java | 28 +
.../DataRegionConfigurationConverter.java | 104 +++
.../config/converters/DiscoverySpiConverter.java | 116 +++
.../converters/SslContextFactoryConverter.java | 81 ++
.../config/ConfigurationConverterTest.java | 217 ++++++
.../migrationtools/config/ConfigExamples.java | 40 +
.../migrationtools/config/ConfigTestUtils.java | 109 +++
.../migration-tools-persistence/build.gradle | 63 ++
.../persistence/BasePersistentTestContext.java | 96 +++
.../CacheDdlCreationFromWorkDirTest.java | 122 +++
.../persistence/JsonSerializersTests.java | 147 ++++
.../migrationtools/persistence/ListCachesTest.java | 56 ++
.../persistence/PdsNodeDiscoveryTests.java | 89 +++
.../persistence/Ignite2PersistenceTools.java | 165 ++++
.../persistence/Ignite2PersistentCacheTools.java | 222 ++++++
.../migrationtools/persistence/LoggerBridge.java | 93 +++
.../persistence/MigrationCacheProcessor.java | 297 ++++++++
.../persistence/MigrationKernal.java | 60 ++
.../persistence/MigrationKernalContext.java | 306 ++++++++
.../persistence/MigrationNodeFolderResolver.java | 124 +++
.../mappers/AbstractSchemaColumnsProcessor.java | 443 +++++++++++
.../persistence/mappers/CacheDataRowProcessor.java | 67 ++
.../persistence/mappers/CustomTupleImpl.java | 261 +++++++
.../IgnoreMismatchesSchemaColumnProcessor.java | 93 +++
.../RecordAndTableSchemaMismatchException.java | 53 ++
.../mappers/SchemaColumnProcessorStats.java | 40 +
.../mappers/SchemaColumnsProcessor.java | 28 +
.../mappers/SimpleSchemaColumnsProcessor.java | 55 ++
.../mappers/SkipRecordsSchemaColumnsProcessor.java | 84 ++
.../persistence/utils/pubsub/BasicProcessor.java | 62 ++
.../utils/pubsub/RateLimiterProcessor.java | 72 ++
.../utils/pubsub/StreamerPublisher.java | 122 +++
.../persistence/SchemaColumnsProcessorTest.java | 484 ++++++++++++
.../utils/pubsub/RateLimiterProcessorTest.java | 68 ++
migration-tools/packaging/cli/build.gradle | 163 ++++
migration-tools/packaging/cli/docker/Dockerfile | 13 +
migration-tools/packaging/cli/start.bat | 51 ++
migration-tools/packaging/cli/start.sh | 43 ++
.../packaging/common/linux/jvmdefaults.sh | 80 ++
.../packaging/common/linux/setup-java.sh | 62 ++
.../packaging/common/windows/jvmdefaults.bat | 67 ++
.../packaging/common/windows/setup-java.bat | 63 ++
.../resources/configs-custom/empty-config.xml | 11 +
.../resources/configs-custom/ignite-config.0.xml | 137 ++++
.../resources/configs-custom/ignite-config.1.xml | 56 ++
.../resources/configs-custom/ignite-config.2.xml | 67 ++
.../resources/sample-clusters/docker-compose.yml | 76 ++
.../sample-clusters/example-persistent-store.xml | 62 ++
.../tools/e2e-tests-framework/.gitignore | 39 +
.../e2e-tests-framework/ai2-runner/.gitignore | 39 +
.../e2e-tests-framework/ai2-runner/build.gradle | 49 ++
.../ai2-runner/docker/Dockerfile | 10 +
.../e2e/framework/runners/Ignite2SeedDataTest.java | 143 ++++
.../framework/runners/Ignite2TestBootStrap.java | 105 +++
.../e2e-tests-framework/framework-core/.gitignore | 38 +
.../framework-core/build.gradle | 31 +
.../tests/e2e/framework/core/DiscoveryUtils.java | 51 ++
.../e2e/framework/core/ExampleBasedCacheTest.java | 56 ++
.../core/ExampleBasedCacheTestProvider.java | 27 +
.../e2e/framework/core/Ignite3TestAnnotation.java | 30 +
.../e2e/framework/core/MyAnnotationProcessor.java | 48 ++
.../tests/e2e/framework/core/SqlTest.java | 28 +
.../tests/e2e/framework/core/SqlTestUtils.java | 69 ++
.../e2e/framework/runners/JdbcTestBootstrap.java | 100 +++
.../e2e-tests-framework/implementations/.gitignore | 38 +
.../implementations/build.gradle | 42 +
.../tests/e2e/impl/IntArrayCachesTest.java | 125 +++
.../tests/e2e/impl/MyBinaryKeyValueCacheTest.java | 110 +++
.../tests/e2e/impl/MyOrganizationsCacheTest.java | 98 +++
.../tests/e2e/impl/MySimpleMapCacheTest.java | 50 ++
.../tests/e2e/impl/PersonCacheTest.java | 58 ++
.../tests/e2e/impl/VeryBasicAbstractCacheTest.java | 164 ++++
settings.gradle | 51 ++
158 files changed, 14580 insertions(+), 1 deletion(-)
diff --git a/build.gradle b/build.gradle
index d0a02aa7f83..e91f9ff7c28 100644
--- a/build.gradle
+++ b/build.gradle
@@ -94,6 +94,25 @@ allprojects {
if (project.hasProperty('excludeTest')) {
exclude project.property('excludeTest')
}
+
+ def dockerVersion = findProperty("migration-tools.docker.version")
+ systemProperty "migration-tools.docker.version", dockerVersion != null ? dockerVersion : project.version
+
+ [
+ "tests.containers.support",
+ "e2e.testLimiter",
+ "ai2.sampleCluster.Xmx",
+ "ai2.sampleCluster.checkpointChecker.maxwaitseconds",
+ "ai2.sampleCluster.checkpointChecker.pollingseconds",
+ "ai2.sampleCluster.recreate.seeding.maxwaitseconds",
+ "ai2.sampleCluster.recreate.seeding.pollingseconds",
+ "seeddata.nCachesPerStint",
+ ].forEach {
+ def v = findProperty(it)
+ if (v != null) {
+ systemProperty it, v
+ }
+ }
}
tasks.withType(Sign).configureEach {
@@ -136,6 +155,8 @@ subprojects {
releasesOnly()
}
}
+
+ mavenLocal()
}
tasks.register('printSubDependencies', DependencyReportTask)
diff --git a/gradle.properties b/gradle.properties
index a496e69783f..2229d5353df 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -34,3 +34,6 @@ projectVersion = 3.1.0-SNAPSHOT
# Versions to use to build DB API Driver wheels.
dbapiPythonVersions = 39,310,311,312,313
+
+ignite2.vendor=org.apache.ignite
+ignite2.version=2.15.0
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 126a451dbc7..9e8894958e3 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -20,6 +20,8 @@ assertj = "3.27.3"
asm = "9.8"
bouncycastle = "1.80"
compileTesting = "0.21.0"
+commons-collections4 = "4.1"
+commons-lang3 = "3.14.0"
fliptables = "1.1.1"
fmpp = "0.9.16"
jackson = "2.19.0"
@@ -32,6 +34,7 @@ jetbrainsAnnotations = "26.0.2"
jline = "3.29.0"
jmh = "1.37"
junit5 = "5.12.2"
+junit5-platform = "1.11.1"
junitPioneer = "2.3.0"
jsr305 = "3.0.2"
okhttp = "4.12.0"
@@ -86,6 +89,7 @@ tree-sitter-hocon = "master-a"
otel = "1.49.0"
spring-boot = "3.4.5"
spring-data = "3.4.5"
+testcontainers = "1.19.0"
#Tools
pmdTool = "6.55.0"
@@ -96,7 +100,7 @@ spotbugsTool = "4.8.3"
[plugins]
openapiGenerator = "org.openapi.generator:7.12.0"
javacc = "com.intershop.gradle.javacc:4.1.3"
-shadow = "io.github.goooler.shadow:8.1.8"
+shadow = "com.gradleup.shadow:8.3.3"
cmake = "dev.welbyseely.gradle-cmake-plugin:0.0.5"
nebula = "com.netflix.nebula.ospackage:11.11.2"
docker = "com.palantir.docker:0.36.0"
@@ -140,6 +144,9 @@ junit5-impl = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref =
junit5-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit5" }
junit-testkit = { module = "org.junit.platform:junit-platform-testkit", version.ref = "testkit" }
+# junit platform
+junit5-platform-console = { module = "org.junit.platform:junit-platform-console", version.ref = "junit5-platform" }
+
#junit-pioneer
junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junitPioneer" }
@@ -178,6 +185,7 @@ jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-dat
typesafe-config = { module = "com.typesafe:config", version.ref = "typesafe" }
+slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
slf4j-jdk14 = { module = "org.slf4j:slf4j-jdk14", version.ref = "slf4j" }
slf4j-log4j = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" }
@@ -279,3 +287,11 @@ spring-boot = { module = "org.springframework.boot:spring-boot", version.ref = "
spring-boot-autoconfigure = { module = "org.springframework.boot:spring-boot-autoconfigure", version.ref = "spring-boot" }
spring-boot-test = { module = "org.springframework.boot:spring-boot-test", version.ref = "spring-boot" }
spring-data-jdbc = { module = "org.springframework.data:spring-data-jdbc", version.ref = "spring-data"}
+
+testcontainers-core = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers"}
+testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers"}
+
+commons-collections4 = { module = "org.apache.commons:commons-collections4", version.ref = "commons-collections4" }
+commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commons-lang3" }
+
+spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version.ref = "spotbugsTool" }
diff --git a/migration-tools/.gitignore b/migration-tools/.gitignore
new file mode 100644
index 00000000000..5f4b054ab09
--- /dev/null
+++ b/migration-tools/.gitignore
@@ -0,0 +1,26 @@
+work
+.DS_Store
+.flattened-pom.xml
+
+.classpath
+.project
+.settings
+
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+
+### IntelliJ IDEA ###
+.idea
+!.idea/codeStyles/codeStyleConfig.xml
+!.idea/codeStyles/Project.xml
+!.idea/checkstyle-idea.xml
+!.idea/inspectionProfiles/Project_Default.xml
+*.iws
+*.iml
+*.ipr
+out/
+
+/resources/sample-clusters/*
+!/resources/sample-clusters/docker-compose.yml
+!/resources/sample-clusters/example-persistent-store.xml
diff --git a/migration-tools/README.md b/migration-tools/README.md
new file mode 100644
index 00000000000..f574934dde9
--- /dev/null
+++ b/migration-tools/README.md
@@ -0,0 +1,198 @@
+# Apache Ignite 3 Migration Tools
+
+This project provides a set of utilities for migrating from Apache Ignite 2 to Apache Ignite 3.
+
+The project is split into two major artifacts:
+
+* [migration-tools-cli](modules%2Fmigration-tools-cli): Packs submodules into a simple CLI tool.
+ * [configuration-converter](modules%2Fconfig-converter):
+ Converts an Apache Ignite 2 configuration file into its Apache Ignite 3 counterpart.
+ * [sql-ddl-generator](modules%2Fsql-ddl-generator):
+ Generates an SQL DDL script for each table in the provided Apache Ignite 2 configuration.
+ * [persistence-tools](modules%2Fmigration-tools-persistence):
+ Allows reading a Apache Ignite 2 node's working directory and migrate the existing data.
+
+## Installing
+
+The tools will be distributed through maven. Currently, they are in the CI/CD build.
+
+### migration-tools-cli
+
+The CLI tools are distributed in a `zip` and `tar.gz` packages.
+
+The executable (`migration-tools`) is in the `bin` folder.
+
+#### Requirements
+
+* Java (any version supported by Ignite 2)
+* Posix shell
+
+### migration-tools-adapter
+
+The **migration tools adapter** can be imported to your maven project as follows:
+
+```xml
+<dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>migration-tools-adapter</artifactId>
+ <version>1.0-SNAPSHOT</version>
+</dependency>
+```
+
+## Usage
+
+### migration-tools-cli
+
+Display help for the available commands:
+`migration-tools --help`
+
+Convert an Apache Ignite 2 configuration file into Apache Ignite 3 cluster and node configurations:
+`migration-tools configuration-converter resources/ignite-config.0.xml resources/node.conf resources/cluster.conf --include-defaults`
+
+Generate an SQL DDL script for each table based on the Apache Ignite 2 file:
+`migration-tools sql-ddl-generator resources/ignite-config.0.xml`
+
+### persistent data
+
+Collection of tools to interact with the Apache Ignite 2 node's work directory.
+
+Examples:
+
+Show `persistent-data` command help:
+`migration-tools persistent-data --help`
+
+#### List Caches
+
+Lists the caches available in the Apache Ignite 2 node's work directory.
+
+##### Examples:
+
+List caches on `node1`:
+
+```shell
+migration-tools \
+ persistent-data \
+ node1/workDir \
+ d270287b-b031-4fdb-abe1-6b124b0be9ef \
+ resources/configs/ignite-config.xml \
+ list-caches
+```
+
+TODO: GG-41668
+```
+Found 2 caches
+CacheName:Country CacheId:-1672482954
+CacheName:City CacheId:2100619
+```
+
+Show `list-caches` command help:
+```shell
+migration-tools \
+ persistent-data \
+ node1/workDir \
+ d270287b-b031-4fdb-abe1-6b124b0be9ef \
+ resources/configs/ignite-config.xml \
+ list-caches \
+ --help
+```
+
+#### Migrate Cache
+
+Migrates data from an Apache Ignite 2 node's directory into a live Apache Ignite 3 cluster.
+The migration is performed partition by partition.
+In case of an error, you may use the saved progress file along with the `--resume-from` option to skip the partitions successfully migrated in the previous run.
+
+The `mode` option defines the error handling policy in case of an irreconcilable mismatch between an Apache Ignite 2 record and the target Apache Ignite 3 table:
+
+* `ABORT` (default): panics and aborts the migration.
+* `IGNORE_COLUMN`: any additional columns/fields in the cache record will be ignored (not migrated to the table; lost). The others will be migrated as usual.
+* `SKIP_RECORD`: The entire cache record will be ignored (not migrated to the table; lost).
+* `PACK_EXTRA`: Any additional columns/fields in the cache record will be serialized to JSON and stored in the `__EXTRA__` column. This is an additional column that the tool adds to the table, it is not a native feature.
+
+The `--rate-limiter N` option limits the number of records migrated per second by using a very basic rate limiter implementation, which may be prone to bursts.
+
+By default, this command will save the progress in the current work directory.
+`--no-save-progress` will not record the progress file.
+
+The `--resume-from progress-file.json` allows resuming the migration from the provided progress file.
+All the partitions marked as successfully migrated in this file will be skipped.
+
+Examples:
+
+Migrate the cache `Country` cache, halting on any error using the `ABORT` mode:
+
+```shell
+migration-tools \
+ persistent-data \
+ node1/workDir \
+ d270287b-b031-4fdb-abe1-6b124b0be9ef \
+ configs/ignite-config.xml \
+ migrate-cache \
+ Country \
+ 127.0.0.1:10800 \
+ --mode ABORT
+```
+
+## Contributing
+
+### Building
+
+The project can be built using maven: `./gradlew build`.
+
+Make sure you meet the following dependencies.
+
+#### Apache Ignite 3 Docker Image
+
+This image is necessary for running tests.
+If the `ignite3.docker.version` property is set to `latest`, you must build the Docker image locally:
+
+1. Clone the [Apache Ignite 3 repository](https://github.com/apache/ignite-3).
+2. Recommended: clone the same revision as declared in the dependencies `ignite3.version.commit`. Automatically get the revision hash by running this in `migration-tools-root`:
+
+ ```bash
+ ./gradlew :properties | awk -F ': ' '/ignite3.version.commit/ {print $2}'
+ ```
+
+3. Build the Docker image (for tests only) : `./gradlew clean :packaging:docker -x check`.
+
+#### Test
+
+The end-to-end tests can take significant time to run.
+You have the option to limit the number of caches for tests using the `-Pe2e.testLimiter=10` property, where `10` is the number of cache examples to test.
+
+##### Test Properties
+
+| Property | Description | Default |
+|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|---------|
+| e2e.testLimiter | Limits the amount of tests run during the end-to-end tests. Can help reduce the build duration. | None |
+| ai2.sampleCluster.Xmx | Adjusts the max JVM memory for the sample cluster containers. | 10g |
+| ai2.sampleCluster.checkpointChecker.maxwaitseconds | Maximum time (s) waiting for checkpoints to happen on the sample cluster. This is used to make sure the cluster shut down gracefully. | 360 |
+| ai2.sampleCluster.checkpointChecker.pollingseconds | Polling period for checking if the checkpoints have completed on the sample clusters. | 10 |
+| ai2.sampleCluster.recreate.seeding.maxwaitseconds | Maximum time (s) spent recreating the sample cluster. The default is very optimistic. It should take around 2 hours. Check below for a faster alternative. | 7200 |
+| ai2.sampleCluster.recreate.seeding.pollingseconds | Polling period for checking if recreating the sample cluster has finished. | 180 |
+| seeddata.nCachesPerStint | Number of caches recreated with the same client instance. Reusing the client helps speeding up the process, but also consumes more resources. | 25 |
+
+##### Sample Clusters
+
+Some of the tests use an Apache Ignite 2 persistent directory.
+Although the framework can recreate the cluster, it can take more than 2 hours.
+As an alternative result, we provide a prebuilt cluster that you can download:
+
+```shell
+bash .ci/download-n-extract-sample-cluster.sh
+```
+
+### Modules
+
+#### table-type-hints
+
+Generates a configuration file that informs the interface adapter of the correct classes for mapping tables/caches.
+This tool inspects the running Apache Ignite 2 cluster to get this information.
+
+Currently, there are no use cases for this module.
+
+### Known Issues
+
+* The messages about properties not converted successfully are hard to read.
+* Enable the license requirements in the checkStyle plugin: TODO: Enable License
+* Remove the checkstyle suppression from the external file.
diff --git a/migration-tools/log4j2-test.xml b/migration-tools/log4j2-test.xml
new file mode 100644
index 00000000000..5154b681e1d
--- /dev/null
+++ b/migration-tools/log4j2-test.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/migration-tools/modules/e2e-ai3-tests/build.gradle b/migration-tools/modules/e2e-ai3-tests/build.gradle
new file mode 100644
index 00000000000..b0887f3fa41
--- /dev/null
+++ b/migration-tools/modules/e2e-ai3-tests/build.gradle
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+apply from: "$rootDir/buildscripts/java-integration-test.gradle"
+
+description = 'migration-tools-e2e-ai3-tests'
+
+dependencies {
+ integrationTestImplementation project(":migration-tools-commons-tests")
+ integrationTestImplementation project(":migration-tools-e2e-core")
+ integrationTestImplementation project(":migration-tools-e2e-implementations")
+ integrationTestImplementation project(":ignite-client")
+ integrationTestImplementation project(":ignite-jdbc")
+ integrationTestImplementation libs.assertj.core
+ integrationTestImplementation libs.junit5.api
+ integrationTestImplementation libs.testcontainers.core
+ integrationTestImplementation libs.testcontainers.junit
+ integrationTestImplementation libs.log4j.api
+ integrationTestImplementation libs.log4j.core
+}
+
+tasks.integrationTest.dependsOn += ":migration-tools-e2e-ai2-runner:docker"
+tasks.integrationTest.dependsOn += ":migration-tools-packaging-cli:docker"
+
diff --git a/migration-tools/modules/e2e-ai3-tests/src/integrationTest/java/org/apache/ignite/migrationtools/tests/e2e/DataMigrationWithIgnite3ClientTest.java b/migration-tools/modules/e2e-ai3-tests/src/integrationTest/java/org/apache/ignite/migrationtools/tests/e2e/DataMigrationWithIgnite3ClientTest.java
new file mode 100644
index 00000000000..c018a279dad
--- /dev/null
+++ b/migration-tools/modules/e2e-ai3-tests/src/integrationTest/java/org/apache/ignite/migrationtools/tests/e2e/DataMigrationWithIgnite3ClientTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.e2e;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
+import static org.junit.jupiter.api.DynamicTest.dynamicTest;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.ignite.client.IgniteClient;
+import org.apache.ignite.migrationtools.tests.bases.MigrationTestBase;
+import org.apache.ignite.migrationtools.tests.e2e.framework.core.DiscoveryUtils;
+import org.apache.ignite.migrationtools.tests.e2e.framework.core.ExampleBasedCacheTest;
+import org.apache.ignite.migrationtools.tests.e2e.framework.core.SqlTest;
+import org.apache.ignite.table.Table;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DynamicNode;
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.TestFactory;
+import org.junit.jupiter.api.function.Executable;
+
+/**
+ * End-to-End test that uses the CLI to execute a cache migration and executes the example tests afterward.
+ * The tests are executed against a AI3 cluster with the native client and the JDBC client.
+ */
+public class DataMigrationWithIgnite3ClientTest extends MigrationTestBase {
+ // TODO: Check if it is possible to rename the tests
+ // TODO: Some of these properties are duplicated with other tests.
+
+ private static final List<ExampleBasedCacheTest> TEST_CLASS_INSTANCES = DiscoveryUtils.discoverClasses();
+
+ private static int N_TEST_EXAMPLES;
+
+ private static int numberOfTestsLimiter;
+
+ @BeforeAll
+ static void setNumTestSamples() {
+ var numTestSamples = System.getenv("N_TEST_SAMPLES");
+ if (numTestSamples != null) {
+ N_TEST_EXAMPLES = Integer.parseUnsignedInt(numTestSamples);
+ } else {
+ N_TEST_EXAMPLES = 2_500;
+ }
+ }
+
+ @BeforeAll
+ static void setNumberOfTestsLimiter() {
+ numberOfTestsLimiter = Integer.parseUnsignedInt(System.getProperty("e2e.testLimiter", String.valueOf(Integer.MAX_VALUE)));
+ }
+
+ @BeforeAll
+ static void enableInfoLogsUnconditionally() {
+ // TODO: Check if this is a good idea. It was quick
+ Configurator.setLevel(LogManager.getLogger(MigrationTestBase.class), Level.INFO);
+ }
+
+ @BeforeAll
+ static void loadJdbcDriver() {
+ ServiceLoader.load(org.apache.ignite.jdbc.IgniteJdbcDriver.class);
+ }
+
+ @TestFactory
+ Stream<DynamicNode> compoundTest() throws Exception {
+ var ai3ClientBuilder = IgniteClient.builder().addresses(MigrationTestBase.AI3_CLUSTER.getAddress());
+ var jdbcAddr = "jdbc:ignite:thin://" + MigrationTestBase.AI3_CLUSTER.getAddress();
+
+ return IntStream.range(0, TEST_CLASS_INSTANCES.size())
+ .mapToObj(i -> {
+ ExampleBasedCacheTest<?, ?> test = TEST_CLASS_INSTANCES.get(i);
+
+ // These assumption are pre-computed because they are the same for all tests, we might as well compute them only once.
+ List<Executable> assumptions = new ArrayList<>();
+ {
+ if (i > numberOfTestsLimiter) {
+ assumptions.add(() -> Assumptions.abort("Number of tests limited to: " + numberOfTestsLimiter));
+ }
+ }
+
+ Function<Executable, Executable> withAssumptions = exec -> () -> {
+ for (Executable ex : assumptions) {
+ ex.execute();
+ }
+
+ exec.execute();
+ };
+
+ Executable migrationTest = withAssumptions.apply(
+ () -> MigrationTestBase.migrationIsSuccessfull(test.getTableName(), "IGNORE_COLUMN")
+ );
+ Executable apiTest = withAssumptions.apply(() -> runIgnite3ApiTest(ai3ClientBuilder, test));
+ Stream<DynamicTest> jdbcTests = test.jdbcTests().entrySet().stream()
+ .map(sqlTestEntry -> {
+ Map.Entry<String, SqlTest> entry = (Map.Entry<String, SqlTest>) sqlTestEntry;
+ String name = entry.getKey();
+ SqlTest sqlTest = entry.getValue();
+ return dynamicTest(name, withAssumptions.apply(() -> runJdbcTest(jdbcAddr, sqlTest)));
+ });
+
+ return dynamicContainer(
+ String.format("T:%d :: %s :: %s", i + 1, test.getClass().getName(), test.getTableName()),
+ Stream.of(
+ dynamicTest("migrationIsSuccessfull", migrationTest),
+ dynamicTest("Ignite 3 java API", apiTest),
+ dynamicContainer("JDBC Tests", jdbcTests)
+ ));
+ });
+ }
+
+ private static void runIgnite3ApiTest(
+ IgniteClient.Builder clientBuilder,
+ ExampleBasedCacheTest<?, ?> exampleBasedTest
+ ) throws Exception {
+ try (IgniteClient client = clientBuilder.build()) {
+ var escapedName = "\"" + exampleBasedTest.getTableName() + "\"";
+ Table table = client.tables().table(escapedName);
+ assertThat(table)
+ .withFailMessage("Table with name must exist: " + escapedName)
+ .isNotNull();
+
+ exampleBasedTest.testIgnite3(table, N_TEST_EXAMPLES);
+ }
+ }
+
+ private static void runJdbcTest(String jdbcConnectionStr, SqlTest sqlTest) throws SQLException {
+ try (var conn = DriverManager.getConnection(jdbcConnectionStr)) {
+ sqlTest.test(conn, N_TEST_EXAMPLES);
+ }
+ }
+}
diff --git a/migration-tools/modules/ignite2-repack/build.gradle b/migration-tools/modules/ignite2-repack/build.gradle
new file mode 100644
index 00000000000..f355233254d
--- /dev/null
+++ b/migration-tools/modules/ignite2-repack/build.gradle
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+ alias(libs.plugins.shadow)
+}
+
+apply plugin: 'java'
+apply plugin: 'signing'
+apply from: "$rootDir/buildscripts/publishing-repos.gradle"
+
+description = 'migration-tools-ignite2-repack'
+
+dependencies {
+ runtimeClasspath libs.ignite2.core
+}
+
+jar {
+ dependsOn(tasks.shadowJar)
+ enabled = false
+}
+
+shadowJar {
+ archiveClassifier.set("")
+ mergeServiceFiles()
+ exclude("ignite.properties")
+ include("*.jar", "org/apache/ignite/IgniteException.class", "org/apache/ignite/binary/**")
+}
+
+publishing {
+ publications {
+ shadow(MavenPublication) {
+ from components.shadow
+ }
+ }
+}
+
+signing {
+ sign publishing.publications
+}
diff --git a/migration-tools/modules/ignite3-repack/build.gradle b/migration-tools/modules/ignite3-repack/build.gradle
new file mode 100644
index 00000000000..b5c95c95e61
--- /dev/null
+++ b/migration-tools/modules/ignite3-repack/build.gradle
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+ alias(libs.plugins.shadow)
+}
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+apply from: "$rootDir/buildscripts/java-test-fixtures.gradle"
+
+description = 'migration-tools-ignite3-repack'
+
+dependencies {
+ runtimeOnly project(":ignite-client")
+ runtimeOnly project(":ignite-api")
+ runtimeOnly project(":ignite-core")
+ runtimeOnly project(":ignite-configuration")
+ runtimeOnly project(":ignite-configuration-api")
+ runtimeOnly project(":ignite-page-memory")
+ runtimeOnly project(":ignite-client-handler")
+ runtimeOnly project(":ignite-runner")
+ runtimeOnly project(":ignite-catalog")
+ runtimeOnly project(":ignite-catalog-dsl")
+ runtimeOnly project(":ignite-network")
+ runtimeOnly project(":ignite-placement-driver-api")
+ runtimeOnly project(":ignite-schema")
+ runtimeOnly project(":ignite-security")
+ runtimeOnly project(":ignite-sql-engine")
+ runtimeOnly project(":ignite-table")
+ runtimeOnly project(":ignite-transactions")
+ runtimeOnly project(":ignite-metastorage-api")
+ runtimeOnly project(":ignite-client-common")
+ runtimeOnly project(":ignite-metrics")
+ runtimeOnly project(":ignite-binary-tuple")
+ runtimeOnly project(":ignite-marshaller-common")
+ runtimeOnly project(":ignite-bytecode")
+ runtimeOnly project(":ignite-network-api")
+ runtimeOnly project(":ignite-vault")
+ runtimeOnly project(":ignite-rocksdb-common")
+ runtimeOnly project(":ignite-replicator")
+ runtimeOnly project(":ignite-raft-api")
+ runtimeOnly project(":ignite-distribution-zones")
+ runtimeOnly project(":ignite-raft")
+ runtimeOnly project(":ignite-partition-distribution")
+ runtimeOnly project(":ignite-cluster-management")
+ runtimeOnly project(":ignite-index")
+ runtimeOnly project(":ignite-storage-api")
+ runtimeOnly project(":ignite-system-view")
+ runtimeOnly project(":ignite-cli")
+
+ // TODO: All the transitive deps should be made into shadow
+ shadow libs.asm.core
+ shadow libs.asm.tree
+ shadow libs.asm.util
+ shadow libs.fastutil.core
+ shadow libs.caffeine
+ shadow libs.netty.common
+ shadow libs.netty.buffer
+ shadow libs.netty.codec
+ shadow libs.netty.handler
+ shadow libs.msgpack.core
+ shadow "org.msgpack:jackson-dataformat-msgpack:0.8.24"
+ shadow libs.auto.service.annotations
+ shadow libs.typesafe.config
+}
+
+jar {
+ dependsOn(tasks.shadowJar)
+ enabled = false
+}
+
+shadowJar {
+ archiveClassifier.set("")
+ //configurations = [project.configurations.implementation]
+ mergeServiceFiles()
+ append 'META-INF/transferable.messages'
+ dependencies {
+ include(dependency(project.group + '::'))
+ }
+ exclude "ignite.properties"
+ relocate('org.apache.ignite', 'org.apache.ignite3')
+}
+
+publishing {
+ publications {
+ shadow(MavenPublication) {
+ from components.shadow
+ }
+ }
+}
+
+signing {
+ sign publishing.publications
+}
diff --git a/migration-tools/modules/ignite3-slim-repack/build.gradle b/migration-tools/modules/ignite3-slim-repack/build.gradle
new file mode 100644
index 00000000000..4f8d9be5854
--- /dev/null
+++ b/migration-tools/modules/ignite3-slim-repack/build.gradle
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+ alias(libs.plugins.shadow)
+}
+
+apply plugin: 'java'
+apply plugin: 'signing'
+apply from: "$rootDir/buildscripts/publishing-repos.gradle"
+
+description = 'migration-tools-ignite3-slim-repack'
+
+dependencies {
+ runtimeOnly project(":ignite-api")
+ runtimeOnly project(":ignite-core")
+ runtimeOnly project(":ignite-configuration")
+ runtimeOnly project(":ignite-configuration-api")
+ runtimeOnly project(":ignite-runner")
+}
+
+jar {
+ dependsOn(tasks.shadowJar)
+ enabled = false
+}
+
+shadowJar {
+ archiveClassifier.set("")
+ mergeServiceFiles()
+ append 'META-INF/transferable.messages'
+ dependencies {
+ include(dependency(project.group + '::'))
+ }
+ exclude "ignite.properties"
+ include(
+ "*.jar",
+ "org/apache/ignite/Ignite.class",
+ "org/apache/ignite/table/IgniteTables.class",
+ "org/apache/ignite/table/Table.class",
+ "org/apache/ignite/table/Tuple.class",
+ "org/apache/ignite/table/KeyValueView.class",
+ "org/apache/ignite/table/DataStreamerTarget.class",
+ "org/apache/ignite/table/manager/IgniteTables.class",
+ "org/apache/ignite/table/mapper/Mapper.class",
+ "org/apache/ignite/table/criteria/CriteriaQuerySource.class",
+ "org/apache/ignite/table/ContinuousQuerySource.class",
+ "org/apache/ignite/table/QualifiedName.class",
+ "org/apache/ignite/compute/ComputeJob.class",
+ "org/apache/ignite/compute/JobExecutionContext.class",
+ "org/apache/ignite/tx/Transaction.class",
+ "org/apache/ignite/marshalling/Marshaller.class",
+ "org/apache/ignite/marshalling/UnsupportedObjectTypeMarshallingException.class",
+ "org/apache/ignite/lang/ErrorGroup.class",
+ "org/apache/ignite/lang/ErrorGroups.class",
+ "org/apache/ignite/lang/IgniteException.class",
+ )
+}
+
+publishing {
+ publications {
+ shadow(MavenPublication) {
+ from components.shadow
+ }
+ }
+}
+
+signing {
+ sign publishing.publications
+}
diff --git a/migration-tools/modules/migration-tools-ai2-test-fixtures/build.gradle b/migration-tools/modules/migration-tools-ai2-test-fixtures/build.gradle
new file mode 100644
index 00000000000..fa9a26f29da
--- /dev/null
+++ b/migration-tools/modules/migration-tools-ai2-test-fixtures/build.gradle
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+apply from: "$rootDir/buildscripts/java-test-fixtures.gradle"
+
+description = 'migration-tools-ai2-test-fixtures'
+
+dependencies {
+ testFixturesImplementation libs.ignite2.core
+}
+
+// TODO: Check if need to add a shaddow jar so that we do not ship the whole ignite 2 core.
diff --git a/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Address.java b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Address.java
new file mode 100644
index 00000000000..159a5ef7197
--- /dev/null
+++ b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Address.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.model;
+
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+
+/**
+ * Employee address.
+ * <p>
+ * This class implements {@link org.apache.ignite.binary.Binarylizable} only for example purposes,
+ * in order to show how to customize serialization and deserialization of
+ * binary objects.
+ */
+public class Address implements Binarylizable {
+ /** Street. */
+ private String street;
+
+ /** ZIP code. */
+ private int zip;
+
+ /**
+ * Required for binary deserialization.
+ */
+ public Address() {
+ // No-op.
+ }
+
+ /**
+ * @param street Street.
+ * @param zip ZIP code.
+ */
+ public Address(String street, int zip) {
+ this.street = street;
+ this.zip = zip;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ writer.writeString("street", street);
+ writer.writeInt("zip", zip);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ street = reader.readString("street");
+ zip = reader.readInt("zip");
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "Address [street=" + street +
+ ", zip=" + zip + ']';
+ }
+}
diff --git a/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Organization.java b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Organization.java
new file mode 100644
index 00000000000..26fe9ff909a
--- /dev/null
+++ b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Organization.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.model;
+
+import java.sql.Timestamp;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+
+/**
+ * This class represents organization object.
+ */
+public class Organization {
+ /** */
+ private static final AtomicLong ID_GEN = new AtomicLong();
+
+ /** Organization ID (indexed). */
+ @QuerySqlField(index = true)
+ private Long id;
+
+ /** Organization name (indexed). */
+ @QuerySqlField(index = true)
+ private String name;
+
+ /** Address. */
+ private Address addr;
+
+ /** Type. */
+ private OrganizationType type;
+
+ /** Last update time. */
+ private Timestamp lastUpdated;
+
+ /**
+ * Required for binary deserialization.
+ */
+ public Organization() {
+ // No-op.
+ }
+
+ /**
+ * @param name Organization name.
+ */
+ public Organization(String name) {
+ id = ID_GEN.incrementAndGet();
+
+ this.name = name;
+ }
+
+ /**
+ * @param id Organization ID.
+ * @param name Organization name.
+ */
+ public Organization(long id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ /**
+ * @param name Name.
+ * @param addr Address.
+ * @param type Type.
+ * @param lastUpdated Last update time.
+ */
+ public Organization(String name, Address addr, OrganizationType type, Timestamp lastUpdated) {
+ id = ID_GEN.incrementAndGet();
+
+ this.name = name;
+ this.addr = addr;
+ this.type = type;
+
+ this.lastUpdated = lastUpdated;
+ }
+
+ /**
+ * @return Organization ID.
+ */
+ public Long id() {
+ return id;
+ }
+
+ /**
+ * @return Name.
+ */
+ public String name() {
+ return name;
+ }
+
+ /**
+ * @return Address.
+ */
+ public Address address() {
+ return addr;
+ }
+
+ /**
+ * @return Type.
+ */
+ public OrganizationType type() {
+ return type;
+ }
+
+ /**
+ * @return Last update time.
+ */
+ public Timestamp lastUpdated() {
+ return lastUpdated;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "Organization [id=" + id +
+ ", name=" + name +
+ ", address=" + addr +
+ ", type=" + type +
+ ", lastUpdated=" + lastUpdated + ']';
+ }
+}
diff --git a/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/OrganizationType.java b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/OrganizationType.java
new file mode 100644
index 00000000000..3b15d3f2e07
--- /dev/null
+++ b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/OrganizationType.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.model;
+
+/**
+ * Organization type enum.
+ */
+public enum OrganizationType {
+ /** Non-profit organization. */
+ NON_PROFIT,
+
+ /** Private organization. */
+ PRIVATE,
+
+ /** Government organization. */
+ GOVERNMENT
+}
diff --git a/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Person.java b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Person.java
new file mode 100644
index 00000000000..d11e26fd8c9
--- /dev/null
+++ b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/Person.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.model;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.cache.affinity.AffinityKey;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.query.annotations.QueryTextField;
+
+/**
+ * Person class.
+ */
+public class Person implements Serializable {
+ /** */
+ private static final AtomicLong ID_GEN = new AtomicLong();
+
+ /** Name of index by two fields (orgId, salary). */
+ public static final String ORG_SALARY_IDX = "ORG_SALARY_IDX";
+
+ /** Person ID (indexed). */
+ @QuerySqlField(index = true)
+ public Long id;
+
+ /** Organization ID (indexed). */
+ @QuerySqlField(index = true, orderedGroups = @QuerySqlField.Group(name = ORG_SALARY_IDX, order = 0))
+ public Long orgId;
+
+ /** First name (not-indexed). */
+ @QuerySqlField
+ public String firstName;
+
+ /** Last name (not indexed). */
+ @QuerySqlField
+ public String lastName;
+
+ /** Resume text (create LUCENE-based TEXT index for this field). */
+ @QueryTextField
+ public String resume;
+
+ /** Salary (indexed). */
+ @QuerySqlField(index = true, orderedGroups = @QuerySqlField.Group(name = ORG_SALARY_IDX, order = 1))
+ public double salary;
+
+ /** Custom cache key to guarantee that person is always colocated with its organization. */
+ private transient AffinityKey<Long> key;
+
+ /**
+ * Default constructor.
+ */
+ public Person() {
+ // No-op.
+ }
+
+ /**
+ * Constructs person record.
+ *
+ * @param org Organization.
+ * @param firstName First name.
+ * @param lastName Last name.
+ * @param salary Salary.
+ * @param resume Resume text.
+ */
+ public Person(Organization org, String firstName, String lastName, double salary, String resume) {
+ // Generate unique ID for this person.
+ id = ID_GEN.incrementAndGet();
+
+ orgId = org.id();
+
+ this.firstName = firstName;
+ this.lastName = lastName;
+ this.salary = salary;
+ this.resume = resume;
+ }
+
+ /**
+ * Constructs person record.
+ *
+ * @param id Person ID.
+ * @param orgId Organization ID.
+ * @param firstName First name.
+ * @param lastName Last name.
+ * @param salary Salary.
+ * @param resume Resume text.
+ */
+ public Person(Long id, Long orgId, String firstName, String lastName, double salary, String resume) {
+ this.id = id;
+ this.orgId = orgId;
+ this.firstName = firstName;
+ this.lastName = lastName;
+ this.salary = salary;
+ this.resume = resume;
+ }
+
+ /**
+ * Constructs person record.
+ *
+ * @param id Person ID.
+ * @param firstName First name.
+ * @param lastName Last name.
+ */
+ public Person(Long id, String firstName, String lastName) {
+ this.id = id;
+
+ this.firstName = firstName;
+ this.lastName = lastName;
+ }
+
+ /**
+ * Gets cache affinity key. Since in some examples person needs to be collocated with organization, we create
+ * custom affinity key to guarantee this collocation.
+ *
+ * @return Custom affinity key to guarantee that person is always collocated with organization.
+ */
+ public AffinityKey<Long> key() {
+ if (key == null)
+ key = new AffinityKey<>(id, orgId);
+
+ return key;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String toString() {
+ return "Person [id=" + id +
+ ", orgId=" + orgId +
+ ", lastName=" + lastName +
+ ", firstName=" + firstName +
+ ", salary=" + salary +
+ ", resume=" + resume + ']';
+ }
+}
diff --git a/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/package-info.java b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/package-info.java
new file mode 100644
index 00000000000..6ad4b795948
--- /dev/null
+++ b/migration-tools/modules/migration-tools-ai2-test-fixtures/src/testFixtures/java/org/apache/ignite/examples/model/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * These classes were copied from Apache Ignite 2 examples to be used as examples in the tests.
+ */
+
+package org.apache.ignite.examples.model;
diff --git a/migration-tools/modules/migration-tools-cli/build.gradle b/migration-tools/modules/migration-tools-cli/build.gradle
new file mode 100644
index 00000000000..d31f3de8622
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/build.gradle
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+
+description = 'migration-tools-cli'
+
+dependencies {
+ implementation project(":migration-tools-commons")
+ implementation project(":migration-tools-config-converter")
+ implementation project(":migration-tools-persistence")
+ implementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ implementation libs.ignite2.core
+ implementation libs.ignite2.spring
+ implementation libs.picocli.core
+ implementation libs.picocli.shell.jline3
+ implementation libs.commons.lang3
+ implementation libs.log4j.core
+ implementation libs.slf4j.jdk14
+
+ testImplementation libs.mockito.core
+ testImplementation libs.assertj.core
+ testImplementation libs.junit5.params
+}
+
+javadoc {
+ exclude 'org/grigain/ignite/migrationtools/cli/**'
+ exclude 'org/apache/ignite/configuration/**'
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/Main.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/Main.java
new file mode 100644
index 00000000000..d390afabfba
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/Main.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.logging.LogManager;
+import org.apache.ignite3.internal.cli.config.CachedConfigManagerProvider;
+import org.apache.ignite3.internal.cli.config.ConfigDefaultValueProvider;
+import org.apache.ignite3.internal.cli.core.exception.handler.PicocliExecutionExceptionHandler;
+import picocli.CommandLine;
+
+/** Main. */
+public class Main {
+ private static final String LOGS_DIR_ENV = "IGNITE_MIGRATION_TOOLS_LOGS_DIR";
+
+ private static final String LOCAL_FOLDER = ".ignite-migration-tools";
+
+ /**
+ * Entry point.
+ *
+ * @param args ignore.
+ */
+ public static void main(String[] args) {
+ initJavaLoggerProps();
+
+ int exitCode = 0;
+ try {
+ exitCode = executeCommand(args);
+ } catch (Exception e) {
+ System.err.println("Error occurred during command execution");
+ }
+
+ System.exit(exitCode);
+ }
+
+ /**
+ * Executes the builds and executes the picocli command.
+ *
+ * @param args Command line arguments.
+ * @return The status code of the command.
+ */
+ public static int executeCommand(String[] args) {
+ CommandLine cmd = new CommandLine(TopLevelCliCommand.class);
+ cmd.setExecutionExceptionHandler(new PicocliExecutionExceptionHandler());
+ cmd.setDefaultValueProvider(new ConfigDefaultValueProvider(new CachedConfigManagerProvider()));
+ cmd.setTrimQuotes(true);
+ cmd.setCaseInsensitiveEnumValuesAllowed(true);
+ return cmd.execute(args);
+ }
+
+ /**
+ * This is a temporary solution to hide unnecessary java util logs that are produced by ivy. ConsoleHandler.level should be set to
+ * SEVERE.
+ */
+ private static void initJavaLoggerProps() {
+ try (InputStream propsFile = Main.class.getResourceAsStream("/cli.java.util.logging.properties")) {
+ if (propsFile != null) {
+ LogManager.getLogManager().updateConfiguration(propsFile, configurationKey -> {
+ // Merge default configuration with configuration read from propsFile
+ // and append the path to logs to the file pattern if propsFile have the corresponding key
+ if (configurationKey.equals("java.util.logging.FileHandler.pattern")) {
+ return (oldConfigValue, newConfigValue) -> {
+ if (newConfigValue == null) {
+ return oldConfigValue;
+ }
+ try {
+ return getLogsDir() + "/" + newConfigValue;
+ } catch (IOException e) {
+ return newConfigValue;
+ }
+ };
+ } else if (configurationKey.equals("java.util.logging.FileHandler.formatter")) {
+ return (oldConfigValue, newConfigValue) -> {
+ String pkgName = "org.apache.ignite";
+ String target = (newConfigValue == null) ? oldConfigValue : newConfigValue;
+ if (target.startsWith(pkgName)) {
+ return pkgName + "3" + target.substring(pkgName.length());
+ } else {
+ return target;
+ }
+ };
+ }
+ return (o, n) -> n == null ? o : n;
+ });
+ }
+ } catch (IOException ignored) {
+ // No-op
+ }
+ }
+
+ private static String getLogsDir() throws IOException {
+ String envLogsDir = System.getenv(LOGS_DIR_ENV);
+ Path logsDir = envLogsDir != null ? Path.of(envLogsDir) : Path.of(System.getProperty("user.home"), LOCAL_FOLDER, "logs");
+ File logsDirFile = logsDir.toFile();
+ if (!logsDirFile.exists()) {
+ if (!logsDirFile.mkdirs()) {
+ throw new IOException("Failed to create directory " + logsDir);
+ }
+ }
+
+ if (logsDirFile.isDirectory()) {
+ return logsDir.toAbsolutePath().toString();
+ } else {
+ throw new IOException(logsDir + " is not a directory");
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/TopLevelCliCommand.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/TopLevelCliCommand.java
new file mode 100644
index 00000000000..748d5aeb020
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/TopLevelCliCommand.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli;
+
+import static org.apache.ignite3.internal.cli.commands.CommandConstants.VERSION_OPTION_ORDER;
+import static org.apache.ignite3.internal.cli.commands.Options.Constants.VERSION_OPTION;
+import static org.apache.ignite3.internal.cli.commands.Options.Constants.VERSION_OPTION_DESC;
+
+import org.apache.ignite.migrationtools.cli.configs.commands.ConfigurationConverterCmd;
+import org.apache.ignite.migrationtools.cli.persistence.commands.PersistenceBaseCmd;
+import org.apache.ignite.migrationtools.cli.sql.commands.SqlDdlGeneratorCmd;
+import org.apache.ignite3.internal.cli.VersionProvider;
+import org.apache.ignite3.internal.cli.commands.BaseCommand;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+/**
+ * Top-level command that prints help and declares subcommands.
+ */
+@Command(name = "migration-tools",
+ versionProvider = VersionProvider.class,
+ description = {
+ "Welcome to Ignite Migration Tools."
+ },
+ subcommands = {
+ CommandLine.HelpCommand.class,
+ ConfigurationConverterCmd.class,
+ SqlDdlGeneratorCmd.class,
+ PersistenceBaseCmd.class
+ })
+public class TopLevelCliCommand extends BaseCommand {
+ @SuppressWarnings("PMD.UnusedPrivateField")
+ @Option(names = VERSION_OPTION, versionHelp = true, description = VERSION_OPTION_DESC, order = VERSION_OPTION_ORDER)
+ private boolean versionRequested;
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/configs/calls/ConfigurationConverterCall.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/configs/calls/ConfigurationConverterCall.java
new file mode 100644
index 00000000000..f0711a6142d
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/configs/calls/ConfigurationConverterCall.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.configs.calls;
+
+import java.nio.file.Path;
+import org.apache.ignite.migrationtools.config.ConfigurationConverter;
+import org.apache.ignite3.internal.cli.core.call.Call;
+import org.apache.ignite3.internal.cli.core.call.CallInput;
+import org.apache.ignite3.internal.cli.core.call.CallOutput;
+import org.apache.ignite3.internal.cli.core.call.DefaultCallOutput;
+
+/** Call to the Configuration Converter. */
+public class ConfigurationConverterCall implements Call<ConfigurationConverterCall.Input, String> {
+
+ @Override
+ public CallOutput<String> execute(Input params) {
+ try {
+ ConfigurationConverter.convertConfigurationFile(
+ params.inputConfigFilePath().toFile(),
+ params.outputNodeConfigFilePath().toFile(),
+ params.outputClusterConfigFilePath().toFile(),
+ params.includeDefaults(),
+ params.clientClassLoader()
+ );
+
+ return DefaultCallOutput.success("");
+ } catch (Exception e) {
+ return DefaultCallOutput.failure(e);
+ }
+ }
+
+ /** Inputs. */
+ public static class Input implements CallInput {
+ private Path inputConfigFilePath;
+
+ private Path outputNodeConfigFilePath;
+
+ private Path outputClusterConfigFilePath;
+
+ private boolean includeDefaults;
+
+ private ClassLoader clientClassLoader;
+
+ /**
+ * Contructor.
+ *
+ * @param inputConfigFilePath InputConfigurationFile
+ * @param outputNodeConfigFilePath Output Node Configuration File Path
+ * @param outputClusterConfigFilePath Output Cluster Configuration File Path
+ * @param includeDefaults Whether defaults should be included in the output.
+ * @param clientClassLoader The classloader to use.
+ */
+ public Input(
+ Path inputConfigFilePath,
+ Path outputNodeConfigFilePath,
+ Path outputClusterConfigFilePath,
+ boolean includeDefaults,
+ ClassLoader clientClassLoader
+ ) {
+ this.inputConfigFilePath = inputConfigFilePath;
+ this.outputNodeConfigFilePath = outputNodeConfigFilePath;
+ this.outputClusterConfigFilePath = outputClusterConfigFilePath;
+ this.includeDefaults = includeDefaults;
+ this.clientClassLoader = clientClassLoader;
+ }
+
+ public Path inputConfigFilePath() {
+ return inputConfigFilePath;
+ }
+
+ public Path outputNodeConfigFilePath() {
+ return outputNodeConfigFilePath;
+ }
+
+ public Path outputClusterConfigFilePath() {
+ return outputClusterConfigFilePath;
+ }
+
+ public boolean includeDefaults() {
+ return includeDefaults;
+ }
+
+ public ClassLoader clientClassLoader() {
+ return clientClassLoader;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/configs/commands/ConfigurationConverterCmd.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/configs/commands/ConfigurationConverterCmd.java
new file mode 100644
index 00000000000..d221a1c1b47
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/configs/commands/ConfigurationConverterCmd.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.configs.commands;
+
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+import org.apache.ignite.migrationtools.cli.configs.calls.ConfigurationConverterCall;
+import org.apache.ignite.migrationtools.cli.mixins.ClassloaderOption;
+import org.apache.ignite3.internal.cli.commands.BaseCommand;
+import org.apache.ignite3.internal.cli.core.call.CallExecutionPipeline;
+import picocli.CommandLine;
+
+/** ConfigurationConverterCmd. */
+@CommandLine.Command(
+ name = "configuration-converter",
+ description = "Converters Ignite 2 configuration xml into Ignite 3 node and cluster configurations")
+public class ConfigurationConverterCmd extends BaseCommand implements Callable<Integer> {
+ @CommandLine.Parameters(paramLabel = "input-file", description = "Ignite 2 Configuration XML")
+ private Path inputFile;
+
+ @CommandLine.Parameters(paramLabel = "node-cfg-output-file", description = "Ignite 3 Configuration HOCON")
+ private Path locCfgFile;
+
+ @CommandLine.Parameters(paramLabel = "cluster-cfg-output-file", description = "Ignite 3 Configuration HOCON")
+ private Path distCfgFile;
+
+ @CommandLine.Option(
+ names = {"-d", "--include-defaults"},
+ description = "Whether default values should also be written to the output files.")
+ private boolean includeDefaults = false;
+
+ @CommandLine.Mixin
+ private ClassloaderOption classloaderOption;
+
+ /** {@inheritDoc} */
+ @Override
+ public Integer call() {
+ var call = new ConfigurationConverterCall();
+ return runPipeline(CallExecutionPipeline.builder(call)
+ .inputProvider(() -> new ConfigurationConverterCall.Input(inputFile, locCfgFile, distCfgFile, includeDefaults,
+ classloaderOption.clientClassLoader()))
+ );
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/exceptions/IgniteClientConnectionExceptionHandler.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/exceptions/IgniteClientConnectionExceptionHandler.java
new file mode 100644
index 00000000000..1072c0c63a3
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/exceptions/IgniteClientConnectionExceptionHandler.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.exceptions;
+
+import org.apache.ignite3.client.IgniteClientConnectionException;
+import org.apache.ignite3.internal.cli.core.exception.ExceptionHandler;
+import org.apache.ignite3.internal.cli.core.exception.ExceptionWriter;
+import org.apache.ignite3.internal.cli.core.style.component.ErrorUiComponent;
+
+/** Simple exception handler for {@link IgniteClientConnectionException}. */
+public class IgniteClientConnectionExceptionHandler implements ExceptionHandler<IgniteClientConnectionException> {
+ @Override
+ public int handle(ExceptionWriter writer, IgniteClientConnectionException e) {
+ writer.write(ErrorUiComponent.fromHeader(e.getMessage()).render());
+ return e.code();
+ }
+
+ @Override
+ public Class<IgniteClientConnectionException> applicableException() {
+ return IgniteClientConnectionException.class;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/mixins/ClassloaderOption.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/mixins/ClassloaderOption.java
new file mode 100644
index 00000000000..9a2e6097803
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/mixins/ClassloaderOption.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.mixins;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import org.apache.ignite.migrationtools.sql.SqlDdlGenerator;
+import picocli.CommandLine;
+
+/** Option that allows the user to add 3rd party libraries to the command custom classloader. */
+public class ClassloaderOption {
+ @CommandLine.Spec
+ private CommandLine.Model.CommandSpec spec;
+
+ private ClassLoader clientClassLoader;
+
+ /**
+ * Parses the command line argument and create the classloader field.
+ *
+ * @param jarFilePaths Cmd line argument.
+ */
+ @CommandLine.Option(
+ paramLabel = "<jarArchive>",
+ names = {"--extra-lib"},
+ description = "Jar files to add to the classpath. To add multiple jars, specify the option multiple times.")
+ public void classloader(Path[] jarFilePaths) {
+ URL[] libraries;
+ if (jarFilePaths != null) {
+ libraries = Arrays.stream(jarFilePaths)
+ .map(p -> {
+ if (!p.getFileName().toString().endsWith(".jar") || !Files.isRegularFile(p)) {
+ throw new CommandLine.ParameterException(spec.commandLine(),
+ String.format("Invalid value '%s' for option '--extra-lib': path does not point to a jar archive.", p));
+ }
+
+ try {
+ return p.toUri().toURL();
+ } catch (MalformedURLException e) {
+ throw new CommandLine.ParameterException(spec.commandLine(),
+ String.format("Invalid value '%s' for option '--extra-lib': path could not be parsed.", p), e);
+ }
+ })
+ .toArray(URL[]::new);
+ } else {
+ libraries = new URL[0];
+ }
+
+ this.clientClassLoader = AccessController.doPrivileged(
+ (PrivilegedAction<URLClassLoader>) () -> new URLClassLoader(libraries, SqlDdlGenerator.class.getClassLoader()));
+ }
+
+ public ClassLoader clientClassLoader() {
+ return clientClassLoader;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/ListCachesCall.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/ListCachesCall.java
new file mode 100644
index 00000000000..9ea620d655c
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/ListCachesCall.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.calls;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.cli.persistence.commands.PersistenceBaseCmd;
+import org.apache.ignite.migrationtools.cli.persistence.params.PersistenceParams;
+import org.apache.ignite.migrationtools.persistence.Ignite2PersistentCacheTools;
+import org.apache.ignite.migrationtools.persistence.MigrationKernalContext;
+import org.apache.ignite3.internal.cli.core.call.Call;
+import org.apache.ignite3.internal.cli.core.call.CallInput;
+import org.apache.ignite3.internal.cli.core.call.DefaultCallOutput;
+import org.apache.ignite3.internal.cli.core.exception.IgniteCliException;
+import org.apache.ignite3.internal.cli.logger.CliLoggers;
+import org.apache.ignite3.internal.logger.IgniteLogger;
+import org.jetbrains.annotations.Nullable;
+
+/** Call to the List Caches Command. */
+public class ListCachesCall implements Call<ListCachesCall.Input, List<ListCachesCall.Output>> {
+ private static final IgniteLogger LOGGER = CliLoggers.forClass(ListCachesCall.class);
+
+ @Override
+ public DefaultCallOutput<List<Output>> execute(Input i) {
+ List<MigrationKernalContext> persistentCtx = Collections.emptyList();
+ try {
+ @Nullable IgniteConfiguration cfg = PersistenceBaseCmd.createValidIgniteCfg(i.persistenceParams());
+ if (cfg == null) {
+ return DefaultCallOutput.failure(new IgniteCliException("Unable to read ignite configuration"));
+ }
+
+ persistentCtx = PersistenceBaseCmd.createAndStartMigrationContext(i.persistenceParams(), cfg);
+ if (persistentCtx.isEmpty()) {
+ return DefaultCallOutput.failure(new IgniteCliException(
+ String.format("Could not find node (consistentId:%s) folder in '%s'", i.persistenceParams().nodeConsistentId(),
+ i.persistenceParams().workDir().toString())
+ ));
+ }
+
+ var cacheIds = Ignite2PersistentCacheTools.persistentCaches(persistentCtx);
+ LOGGER.info("Found {} caches", cacheIds.size());
+
+ var ret = cacheIds.stream()
+ .map(e -> new Output(e.getRight(), e.getLeft()))
+ .collect(Collectors.toList());
+
+ return DefaultCallOutput.success(ret);
+ } catch (IgniteCheckedException e) {
+ return DefaultCallOutput.failure(new IgniteCliException("Error while calling list-caches", e));
+ } finally {
+ for (var ctx : persistentCtx) {
+ try {
+ ctx.stop();
+ } catch (IgniteCheckedException e) {
+ LOGGER.error("Error stopping node context", e);
+ }
+ }
+ }
+ }
+
+ /** Input. */
+ public static class Input implements CallInput {
+ private PersistenceParams persistenceParams;
+
+ public Input(PersistenceParams persistenceParams) {
+ this.persistenceParams = persistenceParams;
+ }
+
+ public PersistenceParams persistenceParams() {
+ return persistenceParams;
+ }
+ }
+
+ /** Output. */
+ public static class Output {
+ private final String cacheName;
+
+ private final int cacheId;
+
+ public Output(String cacheName, int cacheId) {
+ this.cacheName = cacheName;
+ this.cacheId = cacheId;
+ }
+
+ public String cacheName() {
+ return cacheName;
+ }
+
+ public int cacheId() {
+ return cacheId;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/MigrateCacheCall.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/MigrateCacheCall.java
new file mode 100644
index 00000000000..95fed69dbe4
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/MigrateCacheCall.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.calls;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Flow;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.cli.persistence.commands.PersistenceBaseCmd;
+import org.apache.ignite.migrationtools.cli.persistence.params.MigrateCacheParams;
+import org.apache.ignite.migrationtools.cli.persistence.params.MigrationMode;
+import org.apache.ignite.migrationtools.cli.persistence.params.PersistenceParams;
+import org.apache.ignite.migrationtools.persistence.Ignite2PersistentCacheTools;
+import org.apache.ignite.migrationtools.persistence.MigrationKernalContext;
+import org.apache.ignite.migrationtools.persistence.mappers.AbstractSchemaColumnsProcessor;
+import org.apache.ignite.migrationtools.persistence.mappers.IgnoreMismatchesSchemaColumnProcessor;
+import org.apache.ignite.migrationtools.persistence.mappers.SchemaColumnProcessorStats;
+import org.apache.ignite.migrationtools.persistence.mappers.SimpleSchemaColumnsProcessor;
+import org.apache.ignite.migrationtools.persistence.mappers.SkipRecordsSchemaColumnsProcessor;
+import org.apache.ignite.migrationtools.persistence.utils.pubsub.RateLimiterProcessor;
+import org.apache.ignite.migrationtools.sql.SqlDdlGenerator;
+import org.apache.ignite.migrationtools.tablemanagement.PersistentTableTypeRegistryImpl;
+import org.apache.ignite.migrationtools.tablemanagement.RegisterOnlyTableTypeRegistry;
+import org.apache.ignite3.client.IgniteClient;
+import org.apache.ignite3.client.IgniteClientConnectionException;
+import org.apache.ignite3.internal.cli.core.call.Call;
+import org.apache.ignite3.internal.cli.core.call.CallInput;
+import org.apache.ignite3.internal.cli.core.call.CallOutput;
+import org.apache.ignite3.internal.cli.core.call.CallOutputStatus;
+import org.apache.ignite3.internal.cli.core.call.DefaultCallOutput;
+import org.apache.ignite3.internal.cli.core.exception.IgniteCliException;
+import org.apache.ignite3.internal.cli.logger.CliLoggers;
+import org.apache.ignite3.internal.logger.IgniteLogger;
+import org.apache.ignite3.table.DataStreamerItem;
+import org.apache.ignite3.table.Tuple;
+import org.jetbrains.annotations.Nullable;
+
+/** Call to the Migrate Cache command. */
+public class MigrateCacheCall implements Call<MigrateCacheCall.Input, MigrateCacheCall.Ouput> {
+ private static final IgniteLogger LOGGER = CliLoggers.forClass(MigrateCacheCall.class);
+
+ private static Path writeProgressToFile(
+ String nodeConsistentId,
+ String cacheName,
+ List<Map.Entry<Integer, AbstractSchemaColumnsProcessor>> perPartitionColumnProcessors,
+ ProgressData resumeFrom) {
+ // Update the resumeFrom data;
+ perPartitionColumnProcessors.stream()
+ .filter(e -> !e.getValue().hasReceivedError())
+ .map(e -> new PartitionProgressEntry(e.getKey(), e.getValue().getStats()))
+ .forEach(resumeFrom.getCompletedPartitions()::add);
+
+ try {
+ Path progressFileToWrite = Files.createTempFile(Path.of("."),
+ "migrate-cache_" + nodeConsistentId + "_" + cacheName + "_", "_progress.json");
+ try (var os = Files.newOutputStream(progressFileToWrite)) {
+ ObjectMapper objMapper = new ObjectMapper();
+ objMapper.writeValue(os, resumeFrom);
+ }
+ LOGGER.info("ProgressFile saved: {}", progressFileToWrite);
+
+ return progressFileToWrite;
+ } catch (IOException ex) {
+ LOGGER.error("Could not save progress file: {}", ex);
+ return null;
+ }
+ }
+
+ private static ProgressData loadOrInitProgress(Input i) throws InvalidProgressFileException, IOException {
+ String consistentNodeId = i.persistenceParams().nodeConsistentId();
+ String cacheName = i.migrateCacheParams().cacheName();
+
+ if (i.migrateCacheParams().progressFileToRead() != null) {
+ ProgressData resumeFrom;
+ try (var is = Files.newInputStream(i.migrateCacheParams().progressFileToRead())) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ resumeFrom = objectMapper.readValue(is, ProgressData.class);
+ }
+
+ // Check resumeFrom;
+
+ if (!consistentNodeId.equals(resumeFrom.nodeId)) {
+ throw InvalidProgressFileException.forNodeConsistentId(resumeFrom.nodeId, i.persistenceParams().nodeConsistentId());
+ }
+
+ if (!cacheName.equals(resumeFrom.cacheName)) {
+ throw InvalidProgressFileException.forNodeConsistentId(resumeFrom.cacheName, cacheName);
+ }
+
+ var migrationModeStr = i.migrateCacheParams().migrationMode().toString();
+ if (!migrationModeStr.equals(resumeFrom.migrationMode)) {
+ throw InvalidProgressFileException.forMode(resumeFrom.migrationMode, migrationModeStr);
+ }
+
+ return resumeFrom;
+ } else {
+ return new ProgressData(consistentNodeId, cacheName, i.migrateCacheParams().migrationMode().toString(), new ArrayList<>());
+ }
+ }
+
+ private static <T extends AbstractSchemaColumnsProcessor> SchemaColumnProcessorStats collectStats(
+ MigrationMode migrationMode,
+ List<Map.Entry<Integer, T>> schemaColumnsProcessors) {
+ long processedElements = 0;
+ switch (migrationMode) {
+ case SKIP_RECORD:
+ long numSkippedRecords = 0;
+ for (var e : schemaColumnsProcessors) {
+ AbstractSchemaColumnsProcessor p = e.getValue();
+ SkipRecordsSchemaColumnsProcessor.SkippedRecordsStats stats =
+ (SkipRecordsSchemaColumnsProcessor.SkippedRecordsStats) p.getStats();
+ processedElements += stats.getProcessedElements();
+ numSkippedRecords += stats.getNumSkippedRecords();
+ }
+
+ return new SkipRecordsSchemaColumnsProcessor.SkippedRecordsStats(processedElements, numSkippedRecords);
+ case IGNORE_COLUMN:
+ Map<String, Long> droppedCols = new HashMap<>();
+ for (var e : schemaColumnsProcessors) {
+ AbstractSchemaColumnsProcessor p = e.getValue();
+ IgnoreMismatchesSchemaColumnProcessor.IgnoredColumnsStats stats =
+ (IgnoreMismatchesSchemaColumnProcessor.IgnoredColumnsStats) p.getStats();
+ processedElements += stats.getProcessedElements();
+ droppedCols.putAll(stats.getDroppedColumns());
+ }
+
+ return new IgnoreMismatchesSchemaColumnProcessor.IgnoredColumnsStats(processedElements, droppedCols);
+ case PACK_EXTRA:
+ case ABORT:
+ default:
+ for (var e : schemaColumnsProcessors) {
+ AbstractSchemaColumnsProcessor p = e.getValue();
+ SchemaColumnProcessorStats stats = p.getStats();
+ processedElements += stats.getProcessedElements();
+ }
+
+ return new SchemaColumnProcessorStats(processedElements);
+ }
+ }
+
+ @Override
+ public CallOutput<Ouput> execute(Input i) {
+ // TODO: GG-41365 This must be improved. Sometime picocli gets confused with the args.
+ String cacheName = i.migrateCacheParams().cacheName();
+ String[] addresses = i.migrateCacheParams().addresses();
+ if (addresses[0].equals(cacheName)) {
+ addresses = Arrays.copyOfRange(addresses, 1, addresses.length);
+ }
+
+ ProgressData resumeFrom;
+ try {
+ resumeFrom = loadOrInitProgress(i);
+ } catch (InvalidProgressFileException e) {
+ LOGGER.error(e.getMessage());
+ return DefaultCallOutput.failure(e);
+ } catch (IOException e) {
+ LOGGER.error("Could not read from progress file: {}", i.migrateCacheParams().progressFileToRead(), e);
+ return DefaultCallOutput.failure(
+ new IgniteCliException("Cannot read from progress file: " + i.migrateCacheParams().progressFileToRead(), e));
+ }
+
+ List<MigrationKernalContext> persistentCtx = Collections.emptyList();
+ List<Map.Entry<Integer, AbstractSchemaColumnsProcessor>> perPartitionColumnProcessors = new ArrayList<>();
+
+ try (var client = IgniteClient.builder().addresses(addresses).build()) {
+ @Nullable IgniteConfiguration cfg = PersistenceBaseCmd.createValidIgniteCfg(i.persistenceParams());
+ if (cfg == null) {
+ return DefaultCallOutput.failure(new IgniteCliException("Unable to read ignite configuration"));
+ }
+
+ persistentCtx = PersistenceBaseCmd.createAndStartMigrationContext(i.persistenceParams(), cfg);
+ if (persistentCtx.isEmpty()) {
+ return DefaultCallOutput.failure(new IgniteCliException(
+ String.format("Could not find node (consistentId:%s) folder in '%s'", i.persistenceParams().nodeConsistentId(),
+ i.persistenceParams().workDir().toString())
+ ));
+ }
+
+ MigrationMode migrationMode = i.migrateCacheParams().migrationMode();
+ boolean packExtraFields = migrationMode == MigrationMode.PACK_EXTRA;
+ Ignite2PersistentCacheTools.ColumnsProcessorFactory processorFactory =
+ (cacheTuplePublisher, partitionId, clientSchema, columnsToFieldsMappings, typeConverters) -> {
+ // Skip partition
+ if (resumeFrom.getCompletedPartitions().stream().anyMatch(e -> e.getPartitionId() == partitionId)) {
+ return null;
+ }
+
+ Flow.Publisher<DataStreamerItem<Map.Entry<Tuple, Tuple>>> tail;
+ {
+ AbstractSchemaColumnsProcessor processor;
+ switch (migrationMode) {
+ case SKIP_RECORD:
+ processor =
+ new SkipRecordsSchemaColumnsProcessor(clientSchema, columnsToFieldsMappings, typeConverters);
+ break;
+ case IGNORE_COLUMN:
+ processor = new IgnoreMismatchesSchemaColumnProcessor(clientSchema, columnsToFieldsMappings,
+ typeConverters);
+ break;
+ case PACK_EXTRA:
+ case ABORT:
+ default:
+ processor = new SimpleSchemaColumnsProcessor(clientSchema, columnsToFieldsMappings, typeConverters,
+ packExtraFields);
+ }
+ perPartitionColumnProcessors.add(Map.entry(partitionId, processor));
+ cacheTuplePublisher.subscribe(processor);
+ tail = processor;
+ }
+
+ if (i.migrateCacheParams().rateLimiter() > 0) {
+ Flow.Processor<DataStreamerItem<Map.Entry<Tuple, Tuple>>, DataStreamerItem<Map.Entry<Tuple, Tuple>>> rl =
+ new RateLimiterProcessor<>(1, TimeUnit.SECONDS, i.migrateCacheParams().rateLimiter());
+ tail.subscribe(rl);
+ tail = rl;
+ }
+
+ return tail;
+ };
+
+ var registry = new RegisterOnlyTableTypeRegistry(new PersistentTableTypeRegistryImpl(client));
+ var sqlGenerator = new SqlDdlGenerator(registry, packExtraFields);
+
+ // TODO: GG-40802 Add validation. The cache must exist etc.
+ LOGGER.info("Starting the migration process");
+ Ignite2PersistentCacheTools.migrateCache(client, sqlGenerator, persistentCtx, cacheName, processorFactory);
+ LOGGER.info("Waiting for results to be persisted");
+
+ @Nullable Path outputProgressFile;
+ if (!i.migrateCacheParams().saveProgressFileDisabled()) {
+ outputProgressFile = writeProgressToFile(
+ i.persistenceParams().nodeConsistentId(),
+ i.migrateCacheParams().cacheName(),
+ perPartitionColumnProcessors,
+ resumeFrom);
+ } else {
+ outputProgressFile = null;
+ }
+
+ SchemaColumnProcessorStats stats = collectStats(migrationMode, perPartitionColumnProcessors);
+ LOGGER.info("Finished persisting records: {}", stats);
+
+ return DefaultCallOutput.success(new Ouput("Migration finished successfully", outputProgressFile));
+ } catch (Exception e) {
+ // Check if there's a better way to check for connection refused and other on connect errors.
+ if (e instanceof IgniteClientConnectionException && e.getMessage().contains("Client failed to connect")) {
+ LOGGER.error("Could not connect to the cluster", e);
+ return DefaultCallOutput.failure(e);
+ } else {
+ // Lets try to write progress.
+ @Nullable Path outputProgressFile;
+ // TODO: This logic is duplicated.
+ if (!i.migrateCacheParams().saveProgressFileDisabled()) {
+ // Let's just remove the last processor, since there is guarantee that it completed successfully.
+ if (!perPartitionColumnProcessors.isEmpty()) {
+ perPartitionColumnProcessors.remove(perPartitionColumnProcessors.size() - 1);
+ }
+
+ outputProgressFile = writeProgressToFile(
+ i.persistenceParams().nodeConsistentId(),
+ i.migrateCacheParams().cacheName(),
+ perPartitionColumnProcessors,
+ resumeFrom);
+ } else {
+ outputProgressFile = null;
+ }
+
+ LOGGER.error("Error while migration persistence folder", e);
+ return DefaultCallOutput.<Ouput>builder()
+ .status(CallOutputStatus.ERROR)
+ .cause(new IgniteCliException("Error while migrating persistence folder", e))
+ .body(new Ouput("Migration finished unsuccessfully", outputProgressFile))
+ .build();
+ }
+ } finally {
+ for (var ctx : persistentCtx) {
+ try {
+ ctx.stop();
+ } catch (IgniteCheckedException e) {
+ LOGGER.warn("Error while closing persistent contexts", e);
+ }
+ }
+ }
+ }
+
+ static class PartitionProgressEntry {
+ private int partitionId;
+
+ private SchemaColumnProcessorStats stats;
+
+ private PartitionProgressEntry() {
+ super();
+ }
+
+ private PartitionProgressEntry(int partitionId, SchemaColumnProcessorStats stats) {
+ this.partitionId = partitionId;
+ this.stats = stats;
+ }
+
+ public int getPartitionId() {
+ return partitionId;
+ }
+
+ public SchemaColumnProcessorStats getStats() {
+ return stats;
+ }
+ }
+
+ private static class ProgressData {
+ private String nodeId;
+
+ private String cacheName;
+
+ private String migrationMode;
+
+ private List<PartitionProgressEntry> completedPartitions;
+
+ private ProgressData() {
+ super();
+ }
+
+ public ProgressData(String nodeId, String cacheName, String migrationMode, List<PartitionProgressEntry> completedPartitions) {
+ this.nodeId = nodeId;
+ this.cacheName = cacheName;
+ this.migrationMode = migrationMode;
+ this.completedPartitions = completedPartitions;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public String getCacheName() {
+ return cacheName;
+ }
+
+ public String getMigrationMode() {
+ return migrationMode;
+ }
+
+ public List<PartitionProgressEntry> getCompletedPartitions() {
+ return completedPartitions;
+ }
+ }
+
+ /** InvalidProgressFileException. */
+ public static class InvalidProgressFileException extends Exception {
+ public InvalidProgressFileException(String message) {
+ super("Progress file does not match the provided arguments: " + message);
+ }
+
+ static InvalidProgressFileException forNodeConsistentId(String expected, String actual) {
+ return new InvalidProgressFileException("Progress File is from node '" + expected + "', but '" + actual + "' was requested.");
+ }
+
+ static InvalidProgressFileException forCacheName(String expected, String actual) {
+ return new InvalidProgressFileException("Progress File is from cache '" + expected + "', but '" + actual + "' was requested.");
+ }
+
+ static InvalidProgressFileException forMode(String expected, String actual) {
+ return new InvalidProgressFileException(
+ "Progress File has migration mode '" + expected + "', but '" + actual + "' was requested.");
+ }
+ }
+
+ /** Input. */
+ public static class Input implements CallInput {
+ private final PersistenceParams persistenceParams;
+
+ private final MigrateCacheParams migrateCacheParams;
+
+ public Input(PersistenceParams persistenceParams, MigrateCacheParams migrateCacheParams) {
+ this.persistenceParams = persistenceParams;
+ this.migrateCacheParams = migrateCacheParams;
+ }
+
+ public PersistenceParams persistenceParams() {
+ return persistenceParams;
+ }
+
+ public MigrateCacheParams migrateCacheParams() {
+ return migrateCacheParams;
+ }
+ }
+
+ /** Output. */
+ public static class Ouput {
+ private String msg;
+
+ private Path progressFilePath;
+
+ public Ouput(String msg, Path progressFilePath) {
+ this.msg = msg;
+ this.progressFilePath = progressFilePath;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public Path getProgressFilePath() {
+ return progressFilePath;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/RetriableMigrateCacheCall.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/RetriableMigrateCacheCall.java
new file mode 100644
index 00000000000..eea567279cc
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/calls/RetriableMigrateCacheCall.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.calls;
+
+import org.apache.ignite.migrationtools.cli.persistence.params.MigrateCacheParams;
+import org.apache.ignite.migrationtools.cli.persistence.params.PersistenceParams;
+import org.apache.ignite.migrationtools.cli.persistence.params.RetrieableMigrateCacheParams;
+import org.apache.ignite3.internal.cli.core.call.Call;
+import org.apache.ignite3.internal.cli.core.call.CallInput;
+import org.apache.ignite3.internal.cli.core.call.CallOutput;
+import org.apache.ignite3.internal.cli.core.call.DefaultCallOutput;
+import org.apache.ignite3.internal.cli.core.exception.IgniteCliException;
+import org.apache.ignite3.internal.cli.logger.CliLoggers;
+import org.apache.ignite3.internal.logger.IgniteLogger;
+
+/** Call for the Retrieable Migrate Cache Command. */
+public class RetriableMigrateCacheCall implements Call<RetriableMigrateCacheCall.Input, MigrateCacheCall.Ouput> {
+ private static final IgniteLogger LOGGER = CliLoggers.forClass(MigrateCacheCall.class);
+
+ private final MigrateCacheCall migrateCacheCall;
+
+ public RetriableMigrateCacheCall(MigrateCacheCall migrateCacheCall) {
+ this.migrateCacheCall = migrateCacheCall;
+ }
+
+ @Override
+ public CallOutput<MigrateCacheCall.Ouput> execute(Input i) {
+ int retryLimit = i.retryParms().retryLimit();
+ int retryBackoffSeconds = i.retryParms().retryBackoffSeconds();
+
+ if (retryLimit < 0) {
+ LOGGER.error("retryLimit must be >= 0 times, but was {}", retryLimit);
+ return DefaultCallOutput.failure(new IgniteCliException("retryLimit must be >= 0 times, but was " + retryLimit));
+ }
+
+ if (retryBackoffSeconds < 0) {
+ LOGGER.error("retryBackoff must be >= 0 seconds, but was {}", retryBackoffSeconds);
+ return DefaultCallOutput.failure(new IgniteCliException("retryBackoff must be >= 0 seconds, but was " + retryBackoffSeconds));
+ }
+
+ if (retryLimit > 0 && i.migrateCacheParams().saveProgressFileDisabled()) {
+ LOGGER.error("--no-save-progress cannot be used with --retryLimit > 0");
+ return DefaultCallOutput.failure(new IgniteCliException("--no-save-progress cannot be used with --retryLimit > 0"));
+ }
+
+ MigrateCacheParams migrateCacheParams = i.migrateCacheParams();
+
+ int atempt = 0;
+ while (atempt < retryLimit) {
+ CallOutput<MigrateCacheCall.Ouput> status = migrateCacheCall.execute(
+ new MigrateCacheCall.Input(i.persistenceParams(), migrateCacheParams));
+ if (status.errorCause() == null || !status.errorCause().getMessage().equals("Error while migrating persistence folder")) {
+ return status;
+ }
+
+ LOGGER.warn("Cache migration attempt {} failed. Will retry in {}s. {} attempts remaining.",
+ atempt, retryBackoffSeconds, retryLimit - atempt);
+
+ if (retryBackoffSeconds > 0) {
+ try {
+ Thread.sleep(retryBackoffSeconds * 1_000L);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOGGER.warn("Interrupted during retry attempt backoff.");
+ return DefaultCallOutput.failure(new IgniteCliException("Interrupted during retry attempt backoff.", e));
+ }
+ }
+
+ atempt++;
+
+ // make the next call resumeFrom the progressFile of the previous attempt.
+ migrateCacheParams = new MigrateCacheParams(
+ migrateCacheParams.cacheName(),
+ migrateCacheParams.addresses(),
+ migrateCacheParams.migrationMode(),
+ migrateCacheParams.rateLimiter(),
+ migrateCacheParams.saveProgressFileDisabled(),
+ status.body().getProgressFilePath()
+ );
+ }
+
+ CallOutput<MigrateCacheCall.Ouput> lastStatus = migrateCacheCall.execute(
+ new MigrateCacheCall.Input(i.persistenceParams(), migrateCacheParams));
+
+ return lastStatus;
+ }
+
+ /** Inputs. */
+ public static class Input implements CallInput {
+ private final PersistenceParams persistenceParams;
+
+ private final MigrateCacheParams migrateCacheParams;
+
+ private final RetrieableMigrateCacheParams retryParms;
+
+ /**
+ * Constructor.
+ *
+ * @param persistenceParams Persistence Params.
+ * @param migrateCacheParams Migrate cache parameters.
+ * @param retryParms Retry parameters.
+ */
+ public Input(PersistenceParams persistenceParams, MigrateCacheParams migrateCacheParams, RetrieableMigrateCacheParams retryParms) {
+ this.persistenceParams = persistenceParams;
+ this.migrateCacheParams = migrateCacheParams;
+ this.retryParms = retryParms;
+ }
+
+ public PersistenceParams persistenceParams() {
+ return persistenceParams;
+ }
+
+ public MigrateCacheParams migrateCacheParams() {
+ return migrateCacheParams;
+ }
+
+ public RetrieableMigrateCacheParams retryParms() {
+ return retryParms;
+ }
+ }
+
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/ListCachesCmd.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/ListCachesCmd.java
new file mode 100644
index 00000000000..c529f46d811
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/ListCachesCmd.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.commands;
+
+import java.util.concurrent.Callable;
+import org.apache.ignite.migrationtools.cli.persistence.calls.ListCachesCall;
+import org.apache.ignite3.internal.cli.commands.BaseCommand;
+import org.apache.ignite3.internal.cli.core.call.CallExecutionPipeline;
+import org.apache.ignite3.internal.cli.util.PlainTableRenderer;
+import picocli.CommandLine;
+
+/** List Caches command. */
+@CommandLine.Command(
+ name = "list-caches",
+ description = "List available caches on the node"
+)
+public class ListCachesCmd extends BaseCommand implements Callable<Integer> {
+ @CommandLine.ParentCommand
+ private PersistenceBaseCmd parent;
+
+ @Override
+ public Integer call() {
+ var call = new ListCachesCall();
+ return runPipeline(CallExecutionPipeline.builder(call)
+ .inputProvider(() -> new ListCachesCall.Input(parent.params()))
+ .decorator(data -> () -> PlainTableRenderer.render(new String[] {"id", "name"},
+ data.stream().map(o -> new String[] {String.valueOf(o.cacheId()), o.cacheName()}).toArray(Object[][]::new)))
+ );
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/MigrateCacheCmd.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/MigrateCacheCmd.java
new file mode 100644
index 00000000000..64f77e9ae97
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/MigrateCacheCmd.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.commands;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ignite.migrationtools.cli.exceptions.IgniteClientConnectionExceptionHandler;
+import org.apache.ignite.migrationtools.cli.persistence.calls.MigrateCacheCall;
+import org.apache.ignite.migrationtools.cli.persistence.calls.RetriableMigrateCacheCall;
+import org.apache.ignite.migrationtools.cli.persistence.params.MigrateCacheParams;
+import org.apache.ignite.migrationtools.cli.persistence.params.RetrieableMigrateCacheParams;
+import org.apache.ignite3.internal.cli.commands.BaseCommand;
+import org.apache.ignite3.internal.cli.core.call.CallExecutionPipeline;
+import picocli.CommandLine;
+
+/** Migrate cache command. */
+@CommandLine.Command(
+ name = "migrate-cache",
+ description = "Migrate a cache from a Ignite 2 work dir into a Ignite 3 cluster"
+)
+public class MigrateCacheCmd extends BaseCommand implements Callable<Integer> {
+ @CommandLine.ParentCommand
+ private PersistenceBaseCmd parent;
+
+ @CommandLine.Mixin
+ private MigrateCacheParams migrateCacheParams;
+
+ @CommandLine.Mixin
+ private RetrieableMigrateCacheParams retryParams;
+
+ @Override
+ public Integer call() throws Exception {
+ var migrateCacheCall = new MigrateCacheCall();
+ var call = new RetriableMigrateCacheCall(migrateCacheCall);
+ return runPipeline(
+ CallExecutionPipeline.builder(call)
+ .exceptionHandler(new IgniteClientConnectionExceptionHandler())
+ .inputProvider(() -> new RetriableMigrateCacheCall.Input(parent.params(), migrateCacheParams, retryParams))
+ .decorator(ouput -> () -> {
+ List<String> parts = new ArrayList<>(2);
+ if (StringUtils.isNotBlank(ouput.getMsg())) {
+ parts.add(ouput.getMsg());
+ }
+ if (ouput.getProgressFilePath() != null) {
+ parts.add("Progress File: " + ouput.getProgressFilePath());
+ }
+ return String.join("\n", parts);
+ })
+ );
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/PersistenceBaseCmd.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/PersistenceBaseCmd.java
new file mode 100644
index 00000000000..393c327bb62
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/commands/PersistenceBaseCmd.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.commands;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.cli.persistence.params.PersistenceParams;
+import org.apache.ignite.migrationtools.config.Ignite2ConfigurationUtils;
+import org.apache.ignite.migrationtools.persistence.Ignite2PersistenceTools;
+import org.apache.ignite.migrationtools.persistence.MigrationKernalContext;
+import org.apache.ignite3.internal.cli.commands.BaseCommand;
+import org.apache.ignite3.internal.cli.logger.CliLoggers;
+import org.apache.ignite3.internal.logger.IgniteLogger;
+import org.jetbrains.annotations.Nullable;
+import picocli.CommandLine;
+
+/** Base for all the Persistence commands. */
+@CommandLine.Command(
+ name = "persistent-data",
+ description = "Commands to interact with an Ignite 2 node work directory",
+ subcommands = {
+ ListCachesCmd.class,
+ MigrateCacheCmd.class
+ })
+public class PersistenceBaseCmd extends BaseCommand {
+ private static final IgniteLogger LOGGER = CliLoggers.forClass(PersistenceBaseCmd.class);
+
+ @CommandLine.Mixin
+ private PersistenceParams params;
+
+ /**
+ * Creates a valid Ignite configuration.
+ *
+ * @param params The commands param.
+ * @return A valid {@link IgniteConfiguration}, or {@code null} if it doesn't exist.
+ */
+ @Nullable
+ public static IgniteConfiguration createValidIgniteCfg(PersistenceParams params) {
+ var cfg = Ignite2ConfigurationUtils.loadIgnite2Configuration(params.configFile().toFile(), true);
+ // Override the configuration work directory, allows running on a different base folder than the original node.
+ cfg.setWorkDirectory(params.workDir().toString());
+ var errors = MigrationKernalContext.validateConfigurations(cfg);
+ if (!errors.isEmpty()) {
+ LOGGER.error("Could not load all the required configuration modules: %s", errors);
+ return null;
+ }
+
+ return cfg;
+ }
+
+ /**
+ * Creates and starts the {@link MigrationKernalContext}s for the provided params and configurations.
+ *
+ * @param params The command params.
+ * @param cfg The Ignite 2 cluster configuration.
+ * @return A list with the started {@link MigrationKernalContext}s.
+ * @throws IgniteCheckedException if any error occurs.
+ */
+ public static List<MigrationKernalContext> createAndStartMigrationContext(PersistenceParams params, IgniteConfiguration cfg)
+ throws IgniteCheckedException {
+ List<Ignite2PersistenceTools.NodeFolderDescriptor> nodeFolders = Ignite2PersistenceTools.nodeFolderCandidates(cfg).stream()
+ .filter(c -> params.nodeConsistentId().equals(c.consistentId().toString()))
+ .collect(Collectors.toList());
+
+ List<MigrationKernalContext> nodeContexts = new ArrayList<>(nodeFolders.size());
+ for (Ignite2PersistenceTools.NodeFolderDescriptor nodeFolder : nodeFolders) {
+ MigrationKernalContext ctx = new MigrationKernalContext(cfg, nodeFolder.subFolderFile(), nodeFolder.consistentId());
+ nodeContexts.add(ctx);
+
+ ctx.start();
+ }
+
+ return nodeContexts;
+ }
+
+ public PersistenceParams params() {
+ return params;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/MigrateCacheParams.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/MigrateCacheParams.java
new file mode 100644
index 00000000000..faf5658fdd0
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/MigrateCacheParams.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.params;
+
+import java.nio.file.Path;
+import org.jetbrains.annotations.Nullable;
+import picocli.CommandLine;
+
+/** MigrateCacheParams. */
+public class MigrateCacheParams {
+ @CommandLine.Parameters(paramLabel = "cacheName")
+ private String cacheName;
+
+ @CommandLine.Parameters(paramLabel = "urls", arity = "1..*", defaultValue = "127.0.0.1", description = "URLs to connect to the cluster")
+ private String[] addresses;
+
+ @CommandLine.Option(names = "--mode", description = "Mapping error handling policy: ${COMPLETION-CANDIDATES}", defaultValue = "ABORT")
+ private MigrationMode migrationMode;
+
+ @CommandLine.Option(names = {"--rate-limiter"}, defaultValue = "-1", description = "Limits the number of migrated records per second."
+ + " Uses a very basic rate limiter implementation, may be prone to bursts.")
+ private int rateLimiter;
+
+ @CommandLine.Option(names = {"--no-save-progress"}, description = "Disables saving a progress file at the end of the run.")
+ private boolean saveProgressFileDisabled;
+
+ @Nullable
+ @CommandLine.Option(names = {"--resume-from"}, description = "Resumes the migration based on the progress file provided.")
+ private Path progressFileToRead;
+
+ private MigrateCacheParams() {
+ // Intentionally left blank.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param cacheName Cache Name.
+ * @param addresses Address.
+ * @param migrationMode Migration Mode.
+ * @param rateLimiter Rate Limiter
+ * @param saveProgressFileDisabled Save Progress File Enabled.
+ * @param progressFileToRead Progress file to read.
+ */
+ public MigrateCacheParams(
+ String cacheName,
+ String[] addresses,
+ MigrationMode migrationMode,
+ int rateLimiter,
+ boolean saveProgressFileDisabled,
+ @Nullable Path progressFileToRead) {
+ this.cacheName = cacheName;
+ this.addresses = addresses;
+ this.migrationMode = migrationMode;
+ this.rateLimiter = rateLimiter;
+ this.saveProgressFileDisabled = saveProgressFileDisabled;
+ this.progressFileToRead = progressFileToRead;
+ }
+
+ public String cacheName() {
+ return cacheName;
+ }
+
+ public String[] addresses() {
+ return addresses;
+ }
+
+ public MigrationMode migrationMode() {
+ return migrationMode;
+ }
+
+ public int rateLimiter() {
+ return rateLimiter;
+ }
+
+ public boolean saveProgressFileDisabled() {
+ return saveProgressFileDisabled;
+ }
+
+ public Path progressFileToRead() {
+ return progressFileToRead;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/MigrationMode.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/MigrationMode.java
new file mode 100644
index 00000000000..e3f68f3bb8b
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/MigrationMode.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.params;
+
+/**
+ * Allows controlling the error handling policy in case of an irreconcilable mismatch between the GG8 record and the target GG9 Table.
+ */
+public enum MigrationMode {
+ /** Abort the migration. */
+ ABORT,
+ /** The whole cache record will be ignored (not migrated to the table; lost). */
+ SKIP_RECORD,
+ /**
+ * Any additional columns/fields in the cache record will be ignored (not migrated to the table; lost).
+ * The others will be migrated as usual.
+ */
+ IGNORE_COLUMN,
+ /**
+ * Any additional columns/fields in the cache record will be serialized to JSON and stored in the `__EXTRA__` column.
+ * This is an additional column that the tool adds to the table, it is not a native feature.
+ * <p>This mode will allow to leverage the <i>IgniteAdapter.Builder#allowExtraFields</i> feature in the adapter module.</p>
+ */
+ PACK_EXTRA
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/PersistenceParams.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/PersistenceParams.java
new file mode 100644
index 00000000000..1ec4d20fb66
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/PersistenceParams.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.params;
+
+import java.nio.file.Path;
+import picocli.CommandLine;
+
+/** Persistence Command parameters. */
+public class PersistenceParams {
+ @CommandLine.Parameters(paramLabel = "work-dir", description = "Work Directory of one or many Ignite 2 nodes")
+ private Path workDir;
+ @CommandLine.Parameters(paramLabel = "consistent-id", description = "Consistent ID of the Ignite 2 node")
+ private String nodeConsistentId;
+ @CommandLine.Parameters(paramLabel = "config-file", description = "Ignite 2 Configuration XML")
+ private Path configFile;
+
+ public Path workDir() {
+ return workDir;
+ }
+
+ public String nodeConsistentId() {
+ return nodeConsistentId;
+ }
+
+ public Path configFile() {
+ return configFile;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/RetrieableMigrateCacheParams.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/RetrieableMigrateCacheParams.java
new file mode 100644
index 00000000000..e80feb281c5
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/persistence/params/RetrieableMigrateCacheParams.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.params;
+
+import picocli.CommandLine;
+
+/** Parameter for the Retriable Migrate Cache command. */
+public class RetrieableMigrateCacheParams {
+ @CommandLine.Option(names = {"--retryLimit"}, defaultValue = "0",
+ description = "Retries the migration up to N times on retrievable errors. 0 (Default) does not retry."
+ + " Implies save progress is not disabled.")
+ private int retryLimit;
+
+ @CommandLine.Option(names = {"--retryBackoff"}, defaultValue = "0",
+ description = "Waits N seconds before retry the next attempt at migration the cache. Default: 0 (retry immediately).")
+ private int retryBackoffSeconds;
+
+ private RetrieableMigrateCacheParams() {
+ // Intentionally left blank.
+ }
+
+ public RetrieableMigrateCacheParams(int retryLimit, int retryBackoffSeconds) {
+ this.retryLimit = retryLimit;
+ this.retryBackoffSeconds = retryBackoffSeconds;
+ }
+
+ public int retryLimit() {
+ return retryLimit;
+ }
+
+ public int retryBackoffSeconds() {
+ return retryBackoffSeconds;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/sql/calls/SqlDdlGeneratorCall.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/sql/calls/SqlDdlGeneratorCall.java
new file mode 100644
index 00000000000..e8da13acc24
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/sql/calls/SqlDdlGeneratorCall.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.sql.calls;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.cli.sql.commands.SqlDdlGeneratorCmd;
+import org.apache.ignite.migrationtools.config.Ignite2ConfigurationUtils;
+import org.apache.ignite.migrationtools.sql.SqlDdlGenerator;
+import org.apache.ignite.migrationtools.tablemanagement.TableTypeRegistryMapImpl;
+import org.apache.ignite3.catalog.definitions.TableDefinition;
+import org.apache.ignite3.internal.cli.core.call.Call;
+import org.apache.ignite3.internal.cli.core.call.CallInput;
+import org.apache.ignite3.internal.cli.core.call.CallOutput;
+import org.apache.ignite3.internal.cli.core.call.DefaultCallOutput;
+import org.apache.ignite3.internal.cli.core.exception.IgniteCliException;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Call for the SQL Generator command. */
+public class SqlDdlGeneratorCall implements Call<SqlDdlGeneratorCall.Input, String> {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SqlDdlGeneratorCmd.class);
+
+ @Override
+ public CallOutput<String> execute(Input i) {
+ IgniteConfiguration cfg =
+ Ignite2ConfigurationUtils.loadIgnite2Configuration(i.inputFilePath.toFile(), true, i.classloader);
+ CacheConfiguration<?, ?>[] loadedCacheCfgs = Optional.ofNullable(cfg.getCacheConfiguration()).orElse(new CacheConfiguration[0]);
+ if (loadedCacheCfgs.length == 0) {
+ LOGGER.error("Config file does not have any cache configurations: {}", i.inputFilePath);
+ return DefaultCallOutput.failure(
+ new IgniteCliException("Config file does not have any cache configurations: " + i.inputFilePath));
+ }
+
+ List<CacheConfiguration<?, ?>> availableCaches = Arrays.stream(loadedCacheCfgs)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+
+ if (loadedCacheCfgs.length > availableCaches.size()) {
+ var msg = String.format("Loaded %d caches out of %d. Check the errors for more information.", availableCaches.size(),
+ loadedCacheCfgs.length);
+ if (i.stopOnError) {
+ return DefaultCallOutput.failure(new IgniteCliException(msg));
+ } else {
+ LOGGER.warn(msg);
+ }
+ }
+
+ SqlDdlGenerator
+ generator = new SqlDdlGenerator(i.classloader, new TableTypeRegistryMapImpl(), i.allowExtraFields);
+
+ List<TableDefinition> tableDefs = new ArrayList<>(availableCaches.size());
+ for (CacheConfiguration<?, ?> cacheCfg : availableCaches) {
+ try {
+ TableDefinition def = generator.generateTableDefinition(cacheCfg);
+ tableDefs.add(def);
+ } catch (RuntimeException ex) {
+ var msg = "Error while generating table definition for cache: " + cacheCfg.getName();
+ if (i.stopOnError) {
+ return DefaultCallOutput.failure(new IgniteCliException(msg, ex));
+ } else {
+ LOGGER.warn(msg, ex);
+ }
+ }
+ }
+ LOGGER.info("Found definitions for {} caches", tableDefs.size());
+
+ String ddlScript = SqlDdlGenerator.createDdlQuery(tableDefs);
+ LOGGER.info("Finished generating script for caches");
+
+ if (i.targetFilePath == null) {
+ return DefaultCallOutput.success(ddlScript);
+ } else {
+ try (var wr = Files.newBufferedWriter(i.targetFilePath, StandardCharsets.UTF_8)) {
+ wr.write(ddlScript);
+ } catch (IOException e) {
+ LOGGER.error("Error writing to the target file ({}) for writing", i.targetFilePath, e);
+ return DefaultCallOutput.failure(new IgniteCliException(
+ String.format("Error writing to the target file ({}) for writing", i.targetFilePath), e
+ ));
+ }
+
+ return DefaultCallOutput.success("");
+ }
+ }
+
+ /** Inputs. */
+ public static class Input implements CallInput {
+ private Path inputFilePath;
+
+ @Nullable
+ private Path targetFilePath;
+
+ private boolean stopOnError;
+
+ private boolean allowExtraFields;
+
+ private ClassLoader classloader;
+
+ /**
+ * Constructor.
+ *
+ * @param inputFilePath Ignite 2 configuration file.
+ * @param targetFilePath Optional file to save the DDL statements instead of stdout.
+ * @param stopOnError Whether the command should stop on error or just skip caches with errors.
+ * @param allowExtraFields If True, the extra fields column will be added to the tables.
+ * @param classloader {@link ClassLoader} to use during the generation process. May be used to inject 3rd party libraries/classes.
+ */
+ public Input(Path inputFilePath, @Nullable Path targetFilePath, boolean stopOnError, boolean allowExtraFields,
+ ClassLoader classloader) {
+ this.inputFilePath = inputFilePath;
+ this.targetFilePath = targetFilePath;
+ this.stopOnError = stopOnError;
+ this.allowExtraFields = allowExtraFields;
+ this.classloader = classloader;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/sql/commands/SqlDdlGeneratorCmd.java b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/sql/commands/SqlDdlGeneratorCmd.java
new file mode 100644
index 00000000000..0908e927d30
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/java/org/apache/ignite/migrationtools/cli/sql/commands/SqlDdlGeneratorCmd.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.sql.commands;
+
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+import org.apache.ignite.migrationtools.cli.mixins.ClassloaderOption;
+import org.apache.ignite.migrationtools.cli.sql.calls.SqlDdlGeneratorCall;
+import org.apache.ignite3.internal.cli.commands.BaseCommand;
+import org.apache.ignite3.internal.cli.core.call.CallExecutionPipeline;
+import picocli.CommandLine;
+
+/** SQL Generator command. */
+@CommandLine.Command(
+ name = "sql-ddl-generator",
+ description = "Generates a SQL DDL Script from the cache configurations in the input Ignite 2 configuration xml")
+public class SqlDdlGeneratorCmd extends BaseCommand implements Callable<Integer> {
+ @CommandLine.Parameters(paramLabel = "input-file", description = "Ignite 2 Configuration XML")
+ private Path inputFile;
+
+ @CommandLine.Option(names = {"-o", "--output"}, description = "Print the DDL Script to a file instead of the STDOUT")
+ private Path targetFile;
+
+ @CommandLine.Option(
+ names = {"--stop-on-error"},
+ description = "Panics on error. By default, skips errors and prints the SQL DDL Script only for the successful caches.")
+ private boolean stopOnError;
+
+ @CommandLine.Option(
+ names = {"--allow-extra-fields"},
+ description = "Add an extra column to serialize unsupported attributes."
+ + " The resulting tables will be compatible with the 'migrate-cache's 'PACK_EXTRA' mode."
+ + " For more information, please checkout the 'allowExtraFields' method in the IgniteAdapter module.")
+ private boolean allowExtraFields;
+
+ @CommandLine.Mixin
+ private ClassloaderOption classloaderOption;
+
+ /** {@inheritDoc} */
+ @Override
+ public Integer call() {
+ var call = new SqlDdlGeneratorCall();
+ return runPipeline(CallExecutionPipeline.builder(call)
+ .inputProvider(() -> new SqlDdlGeneratorCall.Input(
+ inputFile,
+ targetFile,
+ stopOnError,
+ allowExtraFields,
+ classloaderOption.clientClassLoader()))
+ );
+ }
+}
diff --git a/migration-tools/modules/migration-tools-cli/src/main/resources/log4j2.xml b/migration-tools/modules/migration-tools-cli/src/main/resources/log4j2.xml
new file mode 100644
index 00000000000..2ed6cf35f2c
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/main/resources/log4j2.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<Configuration>
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/migration-tools/modules/migration-tools-cli/src/test/java/org/apache/ignite/migrationtools/cli/persistence/calls/RetriableMigrateCacheCallTest.java b/migration-tools/modules/migration-tools-cli/src/test/java/org/apache/ignite/migrationtools/cli/persistence/calls/RetriableMigrateCacheCallTest.java
new file mode 100644
index 00000000000..871b4c3a20e
--- /dev/null
+++ b/migration-tools/modules/migration-tools-cli/src/test/java/org/apache/ignite/migrationtools/cli/persistence/calls/RetriableMigrateCacheCallTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.cli.persistence.calls;
+
+import static org.apache.ignite3.lang.ErrorGroups.Client.CONNECTION_ERR;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.ignite.migrationtools.cli.persistence.params.MigrateCacheParams;
+import org.apache.ignite.migrationtools.cli.persistence.params.PersistenceParams;
+import org.apache.ignite.migrationtools.cli.persistence.params.RetrieableMigrateCacheParams;
+import org.apache.ignite3.client.IgniteClientConnectionException;
+import org.apache.ignite3.internal.cli.core.call.CallOutput;
+import org.apache.ignite3.internal.cli.core.call.CallOutputStatus;
+import org.apache.ignite3.internal.cli.core.call.DefaultCallOutput;
+import org.apache.ignite3.internal.cli.core.exception.IgniteCliException;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+
+/** Tests for {@link RetriableMigrateCacheCall}. */
+class RetriableMigrateCacheCallTest {
+ private static final Path fakePath = Path.of("/");
+
+ static List<DefaultCallOutput> retriableStatus() {
+ return Collections.singletonList(
+ DefaultCallOutput.<MigrateCacheCall.Ouput>builder()
+ .status(CallOutputStatus.ERROR)
+ .cause(new IgniteCliException("Error while migrating persistence folder"))
+ .body(new MigrateCacheCall.Ouput("Error while migrating persistence folder", fakePath))
+ .build()
+ );
+ }
+
+ static List<DefaultCallOutput> unretriableStatus() {
+ return List.of(
+ DefaultCallOutput.success(new MigrateCacheCall.Ouput("Migration finished successfully", fakePath)),
+ DefaultCallOutput.failure(new MigrateCacheCall.InvalidProgressFileException("Some error message")),
+ DefaultCallOutput.failure(new IgniteCliException("Some random wrapped exception")),
+ DefaultCallOutput.failure(new IgniteClientConnectionException(CONNECTION_ERR, "Client failed to connect", ""))
+ );
+ }
+
+ static List<List<DefaultCallOutput>> execChain4() {
+ List<List<DefaultCallOutput>> statuses = new ArrayList<>();
+ for (var s0 : retriableStatus()) {
+ for (var s1 : retriableStatus()) {
+ for (var s2 : retriableStatus()) {
+ for (var s3 : unretriableStatus()) {
+ var status = List.of(s0, s1, s2, s3);
+ statuses.add(status);
+ }
+ }
+ }
+ }
+ return statuses;
+ }
+
+ static List<List<DefaultCallOutput>> execChain3() {
+ List<List<DefaultCallOutput>> statuses = new ArrayList<>();
+ for (var s0 : retriableStatus()) {
+ for (var s1 : retriableStatus()) {
+ for (var s2 : unretriableStatus()) {
+ var status = List.of(s0, s1, s2);
+ statuses.add(status);
+ }
+ }
+ }
+ return statuses;
+ }
+
+ static List<List<DefaultCallOutput>> execChain2() {
+ List<List<DefaultCallOutput>> statuses = new ArrayList<>();
+ for (var s0 : retriableStatus()) {
+ for (var s1 : unretriableStatus()) {
+ var status = List.of(s0, s1);
+ statuses.add(status);
+ }
+ }
+ return statuses;
+ }
+
+ static List<List<DefaultCallOutput>> execChain1() {
+ List<List<DefaultCallOutput>> statuses = new ArrayList<>();
+ for (var s0 : unretriableStatus()) {
+ var status = List.of(s0);
+ statuses.add(status);
+ }
+ return statuses;
+ }
+
+ @ParameterizedTest(name = "{index}: Status:{0}")
+ @MethodSource({"retriableStatus", "unretriableStatus"})
+ void checkCallWithoutRetries(DefaultCallOutput expectedStatus) {
+ checkCallWithRetries(List.of(expectedStatus), 0, 1);
+ }
+
+ @ParameterizedTest(name = "{index}: RetryLimit:3, ExpectedExecution:1 Status:{0}")
+ @MethodSource("execChain1")
+ void checkLimit3Executions1(List<DefaultCallOutput> execStatus) {
+ checkCallWithRetries(execStatus, 3, 1);
+ }
+
+ @ParameterizedTest(name = "{index}: RetryLimit:3, ExpectedExecution:2 Status:{0}")
+ @MethodSource("execChain2")
+ void checkLimit3Executions2(List<DefaultCallOutput> execStatus) {
+ checkCallWithRetries(execStatus, 3, 2);
+ }
+
+ @ParameterizedTest(name = "{index}: RetryLimit:3, ExpectedExecution:3 Status:{0}")
+ @MethodSource("execChain3")
+ void checkLimit3Executions3(List<DefaultCallOutput> execStatus) {
+ checkCallWithRetries(execStatus, 3, 3);
+ }
+
+ @ParameterizedTest(name = "{index}: RetryLimit:3, ExpectedExecution:4 Status:{0}")
+ @MethodSource("execChain4")
+ void checkWith3RetriesSuccess(List<DefaultCallOutput> execStatus) {
+ checkCallWithRetries(execStatus, 3, 4);
+ }
+
+ void checkCallWithRetries(List<DefaultCallOutput> execStatus, int retryLimit, int numExpectedExecutions) {
+ Iterator<DefaultCallOutput> it = execStatus.iterator();
+ var migrateCacheCall = mock(MigrateCacheCall.class);
+ Mockito.doAnswer(ans -> it.next())
+ .when(migrateCacheCall)
+ .execute(any(MigrateCacheCall.Input.class));
+
+ var c = new RetriableMigrateCacheCall(migrateCacheCall);
+ var retryParams = new RetrieableMigrateCacheParams(retryLimit, 0);
+ CallOutput<MigrateCacheCall.Ouput> ret = c.execute(
+ new RetriableMigrateCacheCall.Input(mock(PersistenceParams.class), mock(MigrateCacheParams.class), retryParams));
+
+ Mockito.verify(migrateCacheCall, times(numExpectedExecutions)).execute(any(MigrateCacheCall.Input.class));
+
+ var expectedStatus = execStatus.get(numExpectedExecutions - 1);
+ assertThat(ret).isSameAs(expectedStatus);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/build.gradle b/migration-tools/modules/migration-tools-commons-tests/build.gradle
new file mode 100644
index 00000000000..184836bf170
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/build.gradle
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+
+description = 'migration-tools-commons-tests'
+
+configurations {
+ unpackDependencies { transitive = false}
+}
+
+dependencies {
+ implementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ implementation libs.commons.lang3
+ implementation libs.testcontainers.core
+ implementation libs.testcontainers.junit
+ implementation libs.assertj.core
+ implementation libs.junit5.api
+ implementation libs.log4j.api
+
+ implementation files(layout.buildDirectory.dir("generated/unpackClassesFromDependencies"))
+
+ unpackDependencies libs.ignite2.core
+}
+
+def unpackTask = tasks.register('unpackClassesFromDependencies', Copy) {
+ includeEmptyDirs = false
+ configurations.unpackDependencies.each { artifact ->
+ from zipTree(artifact)
+ }
+
+ into layout.buildDirectory.dir("generated/$name")
+ include "org/apache/ignite/cache/affinity/AffinityKey.class", "org/apache/ignite/cache/affinity/AffinityKeyMapped.class", "org/apache/ignite/binary/Binarylizable.class"
+}
+
+compileJava {
+ dependsOn unpackTask
+}
+
+/*sourceSets {
+ main {
+ java {
+ compi
+ }
+ compileClasspath layout.buildDirectory.dir("classes/java/main")
+ }
+}*/
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/handlers/SkipUnsupportedOperationsHandlers.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/handlers/SkipUnsupportedOperationsHandlers.java
new file mode 100644
index 00000000000..f0ca3ce09f8
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/handlers/SkipUnsupportedOperationsHandlers.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.handlers;
+
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestExecutionExceptionHandler;
+
+/** Automatically skips tests if based on exceptions thrown for unimplemented features. */
+public class SkipUnsupportedOperationsHandlers implements TestExecutionExceptionHandler {
+ @Override
+ public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
+ if (throwable instanceof UnsupportedOperationException
+ && throwable.getMessage().contains("Only SQL Queries are currently supported")) {
+ // TODO: GG-39659
+ Assumptions.abort("This test requires support for non-SQL queries. (TODO: GG-39659)");
+ return;
+ }
+
+ if (throwable instanceof UnsupportedOperationException
+ && throwable.getMessage().contains("Query with _KEY and _VAL columns is not supported")) {
+ // TODO: GG-40624
+ Assumptions.abort("This test requires support for ai2 internal columns (TODO: GG-40624)");
+ return;
+ }
+
+ if (throwable instanceof RuntimeException
+ && throwable.getMessage().contains("org.apache.calcite.sql.parser.SqlParseException: Encountered \"REGEXP\"")) {
+ // TODO: GG-40623
+ Assumptions.abort("This test requires support for H2 REGEXP operator (TODO: GG-40623)");
+ return;
+ }
+
+ throw throwable;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/bases/MigrationTestBase.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/bases/MigrationTestBase.java
new file mode 100644
index 00000000000..24224c3762a
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/bases/MigrationTestBase.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.bases;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.apache.ignite.migrationtools.tests.containers.Ignite3ClusterContainer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.MountableFile;
+
+/**
+ * Base class for the migration tests.
+ */
+@DisabledIfSystemProperty(
+ named = "tests.containers.support",
+ matches = "false",
+ disabledReason = "Lack of support in TeamCity for testcontainers")
+@ExtendWith(FullSampleCluster.class)
+@Testcontainers
+public class MigrationTestBase {
+ private static final Logger LOGGER = LogManager.getLogger(MigrationTestBase.class);
+
+ public static final String DOCKER_IMAGE_NAME = "ai3-migration-tools:" + System.getProperty("migration-tools.docker.version", "latest");
+
+ private static boolean DEBUG_MODE = Boolean.parseBoolean(System.getProperty("debugMode", "false"));
+
+ private static final Path MIGRATION_TOOLS_CLI_FOLDER = Path.of("target/dependency/migration-tools-cli.jar");
+
+ // TODO: The output of list-caches must be easily parsable so that we can extract the nodeNames
+ protected static final List<String> NODE_NAMES = List.of(
+ "ad26bff6-5ff5-49f1-9a61-425a827953ed",
+ "c1099d16-e7d7-49f4-925c-53329286c444",
+ "7b880b69-8a9e-4b84-b555-250d365e2e67"
+ );
+
+ protected static final Network network = Network.newNetwork();
+
+ // TODO: Add a cluster instead of a single instance. Create a ignite stuff perhaps.
+ @Container
+ protected static final Ignite3ClusterContainer AI3_CLUSTER = new Ignite3ClusterContainer(network);
+
+ @Container
+ protected static final GenericContainer migrationToolsContainer;
+
+ static {
+ // TODO: This should receive the correct version.
+ migrationToolsContainer = new GenericContainer<>(DOCKER_IMAGE_NAME)
+ .withCreateContainerCmdModifier(cmd -> cmd.withEntrypoint("/bin/bash"))
+ .withNetwork(network)
+ .withCommand("-c", "sleep infinity")
+ .withCopyFileToContainer(MountableFile.forHostPath(FullSampleCluster.CLUSTER_CFG_PATH), "/config-file.xml")
+ .withFileSystemBind(FullSampleCluster.TEST_CLUSTER_PATH.toString(), "/storage", BindMode.READ_WRITE)
+ .withEnv("CONFIG_URI", "/config-file.xml");
+
+ if (DEBUG_MODE) {
+ migrationToolsContainer.addEnv("EXTRA_JVM_OPTS", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005");
+ migrationToolsContainer.setPortBindings(Collections.singletonList("5005:5005"));
+ }
+ }
+
+ @AfterAll
+ static void tearDownNetwork() {
+ if (network != null) {
+ network.close();
+ }
+ }
+
+ /** Assert that the migration is successfull. */
+ public static void migrationIsSuccessfull(String cacheName, String migrationMode) throws IOException, InterruptedException {
+ // TODO: Refactor
+ String clusterAddress = "ai3.node.1:10800";
+
+ // migrationArgs.add(1, "");
+ // boolean debugEnabled = Optional.ofNullable(System.getProperty("debug.migration"))
+ // .map(Boolean::parseBoolean)
+ // .orElse(false);
+
+ var logsFolder = Path.of("build/test-logs/");
+ if (!Files.isDirectory(logsFolder)) {
+ Files.createDirectory(logsFolder);
+ }
+
+ for (int i = 0; i < NODE_NAMES.size(); i++) {
+ String nodeId = NODE_NAMES.get(i);
+
+ var fmt = "migration-%s-%d.%s";
+ var logPath = logsFolder.resolve(Path.of(String.format(fmt, cacheName, i, "log")));
+ var stdoutPath = logsFolder.resolve(Path.of(String.format(fmt, cacheName, i, "stdout")));
+ var stderrPath = logsFolder.resolve(Path.of(String.format(fmt, cacheName, i, "stderr")));
+ try (var outStream = Files.newOutputStream(stdoutPath); var errStream = Files.newOutputStream(stderrPath)) {
+ long startTime = System.currentTimeMillis();
+
+ var migrationCmd = migrationToolsContainer.execInContainer(
+ "migration-tools",
+ "persistent-data",
+ "/storage",
+ nodeId,
+ "/config-file.xml",
+ "migrate-cache",
+ cacheName,
+ clusterAddress,
+ "--mode",
+ migrationMode
+ );
+
+ long finishTime = System.currentTimeMillis() - startTime;
+
+ outStream.write(migrationCmd.getStdout().getBytes(StandardCharsets.UTF_8));
+ errStream.write(migrationCmd.getStderr().getBytes(StandardCharsets.UTF_8));
+ migrationToolsContainer.copyFileFromContainer("/root/.ignite-migration-tools/logs/ignite-0.log", logPath.toString());
+
+ assertThat(migrationCmd.getExitCode()).as("Migration command should have finished successfully").isZero();
+
+ LOGGER.info("Finished migrating cache: {};{};{};{}", nodeId, cacheName, migrationMode, finishTime);
+ }
+ }
+ }
+
+
+
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/clusters/FullSampleCluster.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/clusters/FullSampleCluster.java
new file mode 100644
index 00000000000..22129f9441b
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/clusters/FullSampleCluster.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.clusters;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterContainer;
+import org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterWithSamples;
+
+/** Cluster with all the samples from all the caches. */
+public class FullSampleCluster extends Ignite2ClusterWithSamples {
+ public static final Path SAMPLE_CLUSTERS_PATH = Path.of("../../resources/sample-clusters");
+
+ public static final Path CLUSTER_CFG_PATH = SAMPLE_CLUSTERS_PATH.resolve("example-persistent-store.xml");
+
+ public static final Path TEST_CLUSTER_PATH = SAMPLE_CLUSTERS_PATH.resolve("test-cluster");
+
+ public static final List<String> clusterNodeIds = List.of(
+ "ad26bff6-5ff5-49f1-9a61-425a827953ed",
+ "c1099d16-e7d7-49f4-925c-53329286c444",
+ "7b880b69-8a9e-4b84-b555-250d365e2e67"
+ );
+
+ public FullSampleCluster() {
+ super(TEST_CLUSTER_PATH);
+ }
+
+ @Override
+ protected Ignite2ClusterContainer createClusterContainers() {
+ return new Ignite2ClusterContainer(
+ FullSampleCluster.CLUSTER_CFG_PATH,
+ FullSampleCluster.TEST_CLUSTER_PATH,
+ FullSampleCluster.clusterNodeIds
+ );
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterContainer.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterContainer.java
new file mode 100644
index 00000000000..e7e8874eab0
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterContainer.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.containers;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterWithSamples.waitForCondition;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.output.OutputFrame;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.lifecycle.Startable;
+import org.testcontainers.lifecycle.Startables;
+import org.testcontainers.utility.MountableFile;
+
+/** Container of an Ignite 2 cluster. */
+public class Ignite2ClusterContainer implements Startable {
+
+ private static final Logger LOGGER = LogManager.getLogger(Ignite3ClusterContainer.class);
+
+ public final Network network;
+
+ private final List<GenericContainer<?>> containers;
+
+ public Ignite2ClusterContainer(Path cfgFilePath, Path storagePathOnHost, List<String> nodeIds) {
+ this(Network.newNetwork(), cfgFilePath, storagePathOnHost, nodeIds);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param network Network.
+ * @param cfgFilePath Configuration Path.
+ * @param storagePathOnHost Storage path mounted on the host.
+ * @param nodeIds List of node consistent ids.
+ */
+ public Ignite2ClusterContainer(Network network, Path cfgFilePath, Path storagePathOnHost, List<String> nodeIds) {
+ this.network = network;
+ this.containers = new ArrayList<>(nodeIds.size());
+ for (int i = 0; i < nodeIds.size(); i++) {
+ String hostname = "node" + (1 + i);
+ String nodeId = nodeIds.get(i);
+
+ var nodeContainer = createIgnite2Container(
+ network,
+ hostname,
+ nodeId,
+ cfgFilePath,
+ storagePathOnHost
+ );
+
+ containers.add(nodeContainer);
+ }
+
+ // Expose a SQL port on the first node.
+ this.containers.get(0).withExposedPorts(10_800);
+ }
+
+ public Network getNetwork() {
+ return network;
+ }
+
+ private static GenericContainer createIgnite2Container(
+ Network network,
+ String hostName,
+ String nodeId,
+ Path cfgFilePath,
+ Path storagePathOnHost
+ ) {
+ Consumer<OutputFrame> logConsumer = new CheckpointerLogConsumer();
+ String heapSize = System.getProperty("ai2.sampleCluster.Xmx", "10g");
+ return new GenericContainer<>("apacheignite/ignite:2.15.0-jdk11")
+ .withLabel("ai2.sample-cluster.node", hostName)
+ .withNetwork(network)
+ .withNetworkAliases(hostName)
+ .withCopyFileToContainer(MountableFile.forHostPath(cfgFilePath), "/config-file.xml")
+ .withFileSystemBind(storagePathOnHost.toString(), "/storage", BindMode.READ_WRITE)
+ .withEnv("CONFIG_URI", "/config-file.xml")
+ .withEnv("IGNITE_WORK_DIR", "/storage")
+ .withEnv("IGNITE_QUIET", "false")
+ .withEnv("IGNITE_NODE_NAME", nodeId)
+ .withEnv("JVM_OPTS", String.format("-Xmx%s", heapSize))
+ .withLogConsumer(logConsumer)
+ .waitingFor(Wait.forLogMessage(".*Node started .*", 1));
+ }
+
+ @Override
+ public void start() {
+ Startables.deepStart(this.containers).join();
+ }
+
+ @Override
+ public void stop() {
+ doStop(true);
+ }
+
+ /**
+ * Stop the cluster.
+ *
+ * @param waitForCheckpoints Whether should wait for checkpoints to be finished or not.
+ */
+ public void doStop(boolean waitForCheckpoints) {
+ if (waitForCheckpoints) {
+ try {
+ waitForNextCheckpoint();
+ } catch (Exception ex) {
+ LOGGER.error("Error waiting for checkpoints: ", ex);
+ if (ex instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ // TODO: CHMOD This is a hack for team city
+ try {
+ var firstContainer = this.containers.get(0);
+ var chmodOp = firstContainer.execInContainer("chmod", "-R", "777", "/storage");
+ assertThat(chmodOp.getExitCode()).withFailMessage("CHMOD must be successfull").isZero();
+ } catch (IOException ex) {
+ LOGGER.error("Error executing chmod", ex);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ LOGGER.error("Interrupted while executing chmod", ex);
+ }
+
+ // TODO: Close all at the same time.
+ for (GenericContainer<?> node : this.containers) {
+ node.stop();
+ }
+ }
+
+ /**
+ * Waits for the next checkpoint to happen.
+ *
+ * @throws InterruptedException when the thread was interrupted while waiting.
+ */
+ public void waitForNextCheckpoint() throws InterruptedException {
+ // TODO: Make a specific checkpoint log implementation.
+ // TODO: In the future, we will be able to call shutdown(false) on the container to properly shutdown the nodes.
+
+ LOGGER.info("Waiting for checkpoints to happen. This might take a while.");
+ var nodeCheckpointCheckers = this.containers.stream()
+ .map(container -> {
+ String nodeLabel = container.getLabels().get("ai2.sample-cluster.node");
+ CheckpointerLogConsumer logConsumer = (CheckpointerLogConsumer) container.getLogConsumers().get(0);
+ AtomicBoolean checkpointFinished = new AtomicBoolean(false);
+ Runnable onCheckpoint = () -> {
+ LOGGER.info("Checkpoint finished on node: {}", nodeLabel);
+ checkpointFinished.set(true);
+ };
+
+ logConsumer.onCheckpoint(onCheckpoint);
+ return (BooleanSupplier) () -> {
+ if (!container.isRunning() && checkpointFinished.compareAndSet(false, true)) {
+ LOGGER.warn("Container stopped running before the checkpoint was finished: {}", nodeLabel);
+ }
+
+ if (checkpointFinished.get()) {
+ logConsumer.removeListener(onCheckpoint);
+ return true;
+ } else {
+ return false;
+ }
+ };
+ })
+ .collect(toList());
+
+ var maxWaitSeconds = Integer.parseInt(System.getProperty("ai2.sampleCluster.checkpointChecker.maxwaitseconds", "360"));
+ var pollingSeconds = Integer.parseInt(System.getProperty("ai2.sampleCluster.checkpointChecker.pollingseconds", "10"));
+ boolean success = waitForCondition(() -> nodeCheckpointCheckers.stream().allMatch(BooleanSupplier::getAsBoolean),
+ pollingSeconds * 1_000, maxWaitSeconds * 1_000);
+
+ LOGGER.info("Finished waiting for checkpoints: {}", success);
+ }
+
+ private static class CheckpointerLogConsumer implements Consumer<OutputFrame> {
+
+ private List<Runnable> listeners = new CopyOnWriteArrayList<>();
+
+ @Override
+ public void accept(OutputFrame frame) {
+ if (listeners.isEmpty()) {
+ return;
+ }
+
+ if (frame.getType() == OutputFrame.OutputType.STDOUT || frame.getType() == OutputFrame.OutputType.STDERR) {
+ String msg = frame.getUtf8String();
+ if (msg.contains("Checkpoint finished") || msg.contains("Skipping checkpoint (no pages were modified)")) {
+ // Notify listeners
+ for (Runnable listener : listeners) {
+ listener.run();
+ }
+ }
+ }
+ }
+
+ public void onCheckpoint(Runnable action) {
+ this.listeners.add(action);
+ }
+
+ public void removeListener(Runnable action) {
+ this.listeners.remove(action);
+ }
+ }
+
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterWithSamples.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterWithSamples.java
new file mode 100644
index 00000000000..68066c4f8a8
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite2ClusterWithSamples.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.containers;
+
+import static java.lang.Thread.sleep;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.output.OutputFrame;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.MountableFile;
+
+/** Ignite2ClusterWithSamples. */
+public abstract class Ignite2ClusterWithSamples implements BeforeAllCallback {
+
+ private final Path targetPath;
+
+ public Ignite2ClusterWithSamples(Path targetPath) {
+ this.targetPath = targetPath;
+ }
+
+ /** TODO: Copied from IgniteTestUtils should reuse this. */
+ public static boolean waitForCondition(BooleanSupplier cond, long sleepMillis, long timeoutMillis) throws InterruptedException {
+ long stop = System.currentTimeMillis() + timeoutMillis;
+
+ while (System.currentTimeMillis() < stop) {
+ if (cond.getAsBoolean()) {
+ return true;
+ }
+
+ sleep(sleepMillis);
+ }
+
+ return false;
+ }
+
+ private static GenericContainer createIgnite2Container(String nodeId, String nodeName, Network network,
+ Consumer<OutputFrame> logConsumer) {
+ return new GenericContainer<>("apacheignite/ignite:2.15.0-jdk11")
+ .withLabel("ai2.sample-cluster.node", nodeName)
+ .withNetwork(network)
+ .withNetworkAliases(nodeName)
+ .withCopyFileToContainer(MountableFile.forHostPath(FullSampleCluster.CLUSTER_CFG_PATH), "/config-file.xml")
+ .withFileSystemBind(FullSampleCluster.TEST_CLUSTER_PATH.toString(), "/storage", BindMode.READ_WRITE)
+ .withEnv("CONFIG_URI", "/config-file.xml")
+ .withEnv("IGNITE_WORK_DIR", "/storage")
+ .withEnv("IGNITE_QUIET", "false")
+ .withEnv("IGNITE_NODE_NAME", nodeId)
+ .withLogConsumer(logConsumer)
+ .waitingFor(Wait.forLogMessage(".*Node started .*", 1));
+ }
+
+ protected abstract Ignite2ClusterContainer createClusterContainers();
+
+ private void recreateClusterFolder() throws InterruptedException, IOException {
+ // Recreate the model cluster.
+ // TODO: This must be improved.
+ System.out.println("Recreating the sample-cluster");
+ System.out.println("Current Directory; " + Path.of(".").toAbsolutePath().toString());
+
+ Files.createDirectory(targetPath);
+
+ // Get from the outside
+ try (var cluster = createClusterContainers()) {
+ cluster.start();
+
+ // Put the seed data container log in another folder
+ try (var seedLogOs = Files.newOutputStream(targetPath.resolve("seeddata-container.log"))) {
+ Network clusterNetwork = cluster.getNetwork();
+ try (var seedDataContainer = createSeedDataCountainer(clusterNetwork)) {
+ seedDataContainer.start();
+ seedDataContainer.followOutput((Consumer<OutputFrame>) frame -> {
+ if (frame.getType() != OutputFrame.OutputType.END) {
+ try {
+ seedLogOs.write(frame.getBytes());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ // Wait for the container to stop.
+ var maxWaitSeconds = Integer.parseInt(System.getProperty("ai2.sampleCluster.recreate.seeding.maxwaitseconds", "7200"));
+ var pollingSeconds = Integer.parseInt(System.getProperty("ai2.sampleCluster.recreate.seeding.pollingseconds", "180"));
+ boolean success = waitForCondition(
+ () -> !seedDataContainer.isRunning(), pollingSeconds * 1_000, maxWaitSeconds * 1_000);
+
+ // TODO: Status are not working. Should be a bug in test-containers??
+ var state = seedDataContainer.getContainerInfo().getState();
+ System.out.println("Seed data status: " + state);
+
+ Assertions.assertTrue(success, "Timeout waiting for seed data to finish.");
+ Assertions.assertEquals(0, state.getExitCodeLong(), "Seed data container exit code is not 0");
+
+ System.out.println("Finished waiting for seed data container to run");
+ }
+ }
+
+ System.out.println("Stopping the containers");
+ }
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ // TODO: Find a better way to produce the example cluster.
+ // Currently we have to manually create a cluster and run the sample cluster generator in the tools folder.
+ if (!Files.exists(targetPath)) {
+ recreateClusterFolder();
+ }
+ }
+
+ protected GenericContainer createSeedDataCountainer(Network network) {
+ return new GenericContainer("ai3-migration-tools/e2e-tests:latest")
+ .withCopyFileToContainer(MountableFile.forHostPath(FullSampleCluster.CLUSTER_CFG_PATH), "/opt/app/config.xml")
+ .withNetwork(network);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite3ClusterContainer.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite3ClusterContainer.java
new file mode 100644
index 00000000000..3e510a110db
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/containers/Ignite3ClusterContainer.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.containers;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import org.apache.ignite3.client.IgniteClient;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.output.OutputFrame;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.lifecycle.Startable;
+import org.testcontainers.utility.MountableFile;
+
+/** Ignite 3 cluster container. */
+public class Ignite3ClusterContainer implements Startable {
+
+ public static final String DOCKER_IMAGE_NAME = "apacheignite/ignite:" + System.getProperty("ignite3.docker.version", "latest");
+
+ private static Path COMPUTE_LIBS_FOLDER = Path.of("build/dependency");
+
+ private static final Logger LOGGER = LogManager.getLogger(Ignite3ClusterContainer.class);
+
+ public final Network network;
+
+
+ public final GenericContainer node;
+
+ private BufferedWriter logWriter;
+
+ public Ignite3ClusterContainer() {
+ this(Network.newNetwork());
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param network The network to deploy the containers on.
+ */
+ public Ignite3ClusterContainer(Network network) {
+ this.network = network;
+ this.node = new GenericContainer(DOCKER_IMAGE_NAME)
+ .withNetwork(network)
+ .withNetworkAliases("ai3.node.1")
+ .withCommand("--node-name clusterNode1")
+ .withExposedPorts(10800)
+ .waitingFor(Wait.forLogMessage(".*Components started.*", 1));
+ }
+
+ /**
+ * Adds the compute adapter libraries to the container.
+ *
+ * @param extraComputeLibs Extra 3rd party libraries to add.
+ * @return The cluster container.
+ */
+ public Ignite3ClusterContainer withComputeAdapterLibs(Collection<MountableFile> extraComputeLibs) {
+ this.node.addEnv("IGNITE_INTERFACE_ADAPTER_COMPUTE_FOLDER", "/opt/ignite/compute-libs");
+
+ try {
+ // Copy bootstrap class (classworlds, compute-bootstrap)
+ Files.list(COMPUTE_LIBS_FOLDER.resolve("lib"))
+ .map(MountableFile::forHostPath)
+ .forEach(mf -> this.node.withCopyFileToContainer(mf, "/opt/ignite/lib/"));
+
+ // Copy compute libs (cache-api, ignite-core, compute-core)
+ var baseComputeLibs = Files.list(COMPUTE_LIBS_FOLDER.resolve("compute-libs")).map(MountableFile::forHostPath);
+
+ // And extraComputeLibs
+ Stream.concat(baseComputeLibs, extraComputeLibs.stream())
+ .forEach(mf -> this.node.withCopyFileToContainer(mf, "/opt/ignite/compute-libs/"));
+ } catch (IOException ex) {
+ throw new RuntimeException("Problems copying compute libs to container", ex);
+ }
+
+ return this;
+ }
+
+ /** Enables debug. */
+ public Ignite3ClusterContainer withDebug() {
+ this.node.addEnv("IGNITE3_EXTRA_JVM_ARGS", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:8000");
+ this.node.addExposedPort(8000);
+ return this;
+ }
+
+ @Override
+ public void start() {
+ this.node.start();
+
+ Path parentFolder = Path.of("build/test-logs");
+ Path logFilePath = parentFolder.resolve("ignite-3-" + this.node.getContainerId().substring(0, 8));
+ CountDownLatch startupLatch = new CountDownLatch(1);
+
+ try {
+ Files.createDirectories(parentFolder);
+ this.logWriter = Files.newBufferedWriter(logFilePath);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ this.node.followOutput((Consumer<OutputFrame>) frame -> {
+ var type = frame.getType();
+ if (type == OutputFrame.OutputType.STDOUT || type == OutputFrame.OutputType.STDERR) {
+ String msg = frame.getUtf8String();
+ try {
+ this.logWriter.write(msg);
+ } catch (IOException e) {
+ LOGGER.error("Error writing cluster logs to file: ", e);
+ }
+
+ if (msg.contains("[IgniteServerImpl] Apache Ignite started successfully!")) {
+ startupLatch.countDown();
+ }
+ }
+ });
+
+ Container.ExecResult execRes = null;
+ try {
+ execRes = this.node.execInContainer(
+ "/opt/ignite3cli/bin/ignite3",
+ "cluster",
+ "init",
+ "--name=ignite-cluster",
+ "--url=http://127.0.0.1:10300",
+ "--metastorage-group=clusterNode1");
+
+ if (execRes.getExitCode() != 0) {
+ throw new RuntimeException("Could not init cluster: " + execRes.getStderr());
+ }
+
+ boolean started = startupLatch.await(10_000, TimeUnit.MILLISECONDS);
+ if (!started) {
+ throw new RuntimeException("Failed to catch cluster started signal after cluster init");
+ }
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ this.node.stop();
+ if (this.logWriter != null) {
+ try {
+ this.logWriter.close();
+ } catch (IOException e) {
+ LOGGER.error("Exception closing the cluster log file: ", e);
+ }
+ }
+ }
+
+ public String getAddress() {
+ return this.node.getHost() + ":" + this.node.getFirstMappedPort();
+ }
+
+ public IgniteClient.Builder clientBuilder() {
+ return IgniteClient.builder().addresses(getAddress());
+ }
+
+ public IgniteClient buildClient() {
+ return clientBuilder().build();
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/models/ComplexKeyIntStr.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/models/ComplexKeyIntStr.java
new file mode 100644
index 00000000000..bd65f6300de
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/models/ComplexKeyIntStr.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.models;
+
+import java.util.Objects;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
+
+/** ComplexKeyIntStr. */
+public class ComplexKeyIntStr {
+ // TODO: Checkout to change fields casing - https://issues.apache.org/jira/browse/IGNITE-6785
+ private int id;
+
+ @AffinityKeyMapped
+ private String affinityStr;
+
+ public ComplexKeyIntStr(int id, String affinityStr) {
+ this.id = id;
+ this.affinityStr = affinityStr;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public String getAffinityStr() {
+ return affinityStr;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ComplexKeyIntStr str = (ComplexKeyIntStr) o;
+ return id == str.id && Objects.equals(affinityStr, str.affinityStr);
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(id, affinityStr);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/models/SimplePojo.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/models/SimplePojo.java
new file mode 100644
index 00000000000..ba30fe7b4b6
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/models/SimplePojo.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.models;
+
+import java.math.BigDecimal;
+import java.util.Objects;
+
+/** Simple Pojo. */
+public class SimplePojo {
+
+ private String name;
+
+ private int amount;
+
+ private BigDecimal decimalAmount;
+
+ private SimplePojo() {
+ // Default constructor
+ // Intentionally left blank.
+ }
+
+ /** Constructor. */
+ public SimplePojo(String name, int amount, BigDecimal decimalAmount) {
+ this.name = name;
+ this.amount = amount;
+ this.decimalAmount = decimalAmount;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAmount() {
+ return amount;
+ }
+
+ public void setAmount(int amount) {
+ this.amount = amount;
+ }
+
+ public BigDecimal getDecimalAmount() {
+ return decimalAmount;
+ }
+
+ public void setDecimalAmount(BigDecimal decimalAmount) {
+ this.decimalAmount = decimalAmount;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SimplePojo pojo = (SimplePojo) o;
+ return amount == pojo.amount && Objects.equals(name, pojo.name) && Objects.equals(decimalAmount, pojo.decimalAmount);
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(name, amount, decimalAmount);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/utils/RunIgnite2Cluster.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/utils/RunIgnite2Cluster.java
new file mode 100644
index 00000000000..3baa2f3ef1b
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/utils/RunIgnite2Cluster.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.utils;
+
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterContainer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+/** RunIgnite2Cluster. */
+@ExtendWith(FullSampleCluster.class)
+@Testcontainers
+public class RunIgnite2Cluster {
+ private final Ignite2ClusterContainer cluster = new Ignite2ClusterContainer(
+ FullSampleCluster.CLUSTER_CFG_PATH,
+ FullSampleCluster.TEST_CLUSTER_PATH,
+ FullSampleCluster.clusterNodeIds
+ );
+
+ @Test
+ void runIgnite2ContainerWithSamples() {
+ try (cluster) {
+ cluster.start();
+ System.out.println("Cluster started. Feel free to pause and debug.");
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/utils/RunIgnite3Cluster.java b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/utils/RunIgnite3Cluster.java
new file mode 100644
index 00000000000..0d225614639
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons-tests/src/main/java/org/apache/ignite/migrationtools/tests/utils/RunIgnite3Cluster.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tests.utils;
+
+import org.apache.ignite.migrationtools.tests.containers.Ignite3ClusterContainer;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+/** RunIgnite3Cluster. */
+@Testcontainers
+public class RunIgnite3Cluster {
+
+ private final Ignite3ClusterContainer cluster = new Ignite3ClusterContainer();
+
+ @Test
+ void runIgnite3() {
+ try (cluster) {
+ cluster.start();
+ System.out.println("Cluster started. Feel free to pause and debug.");
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/build.gradle b/migration-tools/modules/migration-tools-commons/build.gradle
new file mode 100644
index 00000000000..ec7942bcece
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/build.gradle
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+
+description = 'migration-tools-commons'
+
+dependencies {
+ implementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ implementation libs.ignite2.core
+ implementation libs.ignite2.spring
+ implementation libs.commons.lang3
+ implementation libs.commons.collections4
+ implementation libs.slf4j.api
+
+ testImplementation project(":migration-tools-commons-tests")
+ testImplementation testFixtures(project(":migration-tools-ai2-test-fixtures"))
+ testImplementation libs.assertj.core
+ testImplementation libs.junit5.api
+ testImplementation libs.junit5.params
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/Ignite2ConfigurationUtils.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/Ignite2ConfigurationUtils.java
new file mode 100644
index 00000000000..85101ceba18
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/Ignite2ConfigurationUtils.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import java.beans.PropertyDescriptor;
+import java.io.File;
+import org.apache.commons.collections4.MultiValuedMap;
+import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.loader.CustomBeanFactory;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.beans.factory.support.DefaultListableBeanFactory;
+import org.springframework.beans.factory.xml.XmlBeanDefinitionReader;
+import org.springframework.context.support.GenericApplicationContext;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.core.io.Resource;
+
+/**
+ * Utility methods for managing Ignite 2 Configurations.
+ */
+public class Ignite2ConfigurationUtils {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Ignite2ConfigurationUtils.class);
+
+ private Ignite2ConfigurationUtils() {
+ // Intentionally left blank
+ }
+
+ public static IgniteConfiguration loadIgnite2Configuration(File inputFile, boolean skipUnloadableBeans) {
+ return loadIgnite2Configuration(new FileSystemResource(inputFile), createBeanFactory(skipUnloadableBeans), null);
+ }
+
+ public static IgniteConfiguration loadIgnite2Configuration(File inputFile, boolean skipUnloadableBeans,
+ @Nullable ClassLoader clientClassLoader) {
+ return loadIgnite2Configuration(new FileSystemResource(inputFile), createBeanFactory(skipUnloadableBeans), clientClassLoader);
+ }
+
+ /**
+ * Load and Apache Ignite 2 configuration into memory.
+ *
+ * @param rsrc The input xml configuration file as a resource.
+ * @param factory The BeanFactory
+ * @return The IgniteConfiguration object.
+ */
+ public static IgniteConfiguration loadIgnite2Configuration(
+ Resource rsrc,
+ DefaultListableBeanFactory factory,
+ @Nullable ClassLoader classLoader
+ ) throws BeansException, IllegalStateException {
+
+ // Get the Spring Bean Def into memory
+ GenericApplicationContext springCtx = new GenericApplicationContext(factory);
+ springCtx.setClassLoader(classLoader);
+ XmlBeanDefinitionReader reader = new XmlBeanDefinitionReader(springCtx);
+
+ reader.setValidationMode(XmlBeanDefinitionReader.VALIDATION_XSD);
+
+ var n = reader.loadBeanDefinitions(rsrc);
+ LOGGER.info("Loaded Spring definition into memory: {}", n);
+
+ springCtx.refresh();
+
+ return factory.getBean(IgniteConfiguration.class);
+ }
+
+ /**
+ * Create spring bean factory based on the skipUnloadableBeans property.
+ *
+ * @param skipUnloadableBeans Whether the factory should skip unloadable beans or fail with an error.
+ * @return The appropriate bean factory.
+ */
+ public static DefaultListableBeanFactory createBeanFactory(boolean skipUnloadableBeans) {
+ if (skipUnloadableBeans) {
+ MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> propertyTracker = new HashSetValuedHashMap<>();
+ return new CustomBeanFactory(propertyTracker);
+ } else {
+ return new DefaultListableBeanFactory();
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/Ignite3ConfigurationUtils.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/Ignite3ConfigurationUtils.java
new file mode 100644
index 00000000000..0a768742a54
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/Ignite3ConfigurationUtils.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.ignite.migrationtools.config.registry.CombinedConfigRegistry;
+import org.apache.ignite.migrationtools.config.storage.NoDefaultsStorageConfiguration;
+import org.apache.ignite3.configuration.ConfigurationModule;
+import org.apache.ignite3.configuration.RootKey;
+import org.apache.ignite3.configuration.annotation.ConfigurationType;
+import org.apache.ignite3.configuration.validation.Validator;
+import org.apache.ignite3.internal.configuration.ConfigurationManager;
+import org.apache.ignite3.internal.configuration.ConfigurationModules;
+import org.apache.ignite3.internal.configuration.ConfigurationRegistry;
+import org.apache.ignite3.internal.configuration.ConfigurationTreeGenerator;
+import org.apache.ignite3.internal.configuration.ServiceLoaderModulesProvider;
+import org.apache.ignite3.internal.configuration.storage.LocalFileConfigurationStorage;
+import org.apache.ignite3.internal.configuration.validation.ConfigurationValidator;
+import org.apache.ignite3.internal.configuration.validation.ConfigurationValidatorImpl;
+import org.apache.ignite3.internal.manager.ComponentContext;
+import org.apache.ignite3.internal.security.authentication.validator.AuthenticationProvidersValidatorImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility methods for loading Ignite 3 Configuration Modules.
+ * TODO: This class was heavily adapted from the Ignite Runner.
+ */
+public class Ignite3ConfigurationUtils {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Ignite3ConfigurationUtils.class);
+
+ /**
+ * Loads a special combined registry with modules from both the Cluster and Node modules.
+ *
+ * @param nodeCfgPath Node configuration path.
+ * @param clusterCfgPath Cluster configuration path.
+ * @param includeDefaults Include defaults.
+ */
+ public static CombinedConfigRegistry loadCombinedRegistry(Path nodeCfgPath, Path clusterCfgPath, boolean includeDefaults) {
+ var locReg = loadNodeConfiguration(nodeCfgPath, includeDefaults);
+ var distReg = loadClusterConfiguration(clusterCfgPath, includeDefaults);
+ return new CombinedConfigRegistry(locReg, distReg);
+ }
+
+ /**
+ * Loads a Configuration Registry with only Node modules.
+ *
+ * @param cfgPath Configuration path.
+ * @param includeDefaults Include defaults.
+ */
+ public static ConfigurationRegistry loadNodeConfiguration(Path cfgPath, boolean includeDefaults) {
+ return loadConfigurations(cfgPath, loadConfigurationModules().local(), includeDefaults);
+ }
+
+ /**
+ * Loads a Configuration Registry with only Cluster modules.
+ *
+ * @param cfgPath Config path.
+ * @param includeDefaults Include defaults.
+ */
+ public static ConfigurationRegistry loadClusterConfiguration(Path cfgPath, boolean includeDefaults) {
+ // Hack so that it passes the validation
+ // TODO: This is another hack that needs to be cleaned. We don't really need the ConfigurationRegistry.
+ var distributedModule = loadConfigurationModules().distributed();
+ for (RootKey<?, ?> key : distributedModule.rootKeys()) {
+ try {
+ FieldUtils.writeDeclaredField(key, "storageType", ConfigurationType.LOCAL, true);
+ } catch (IllegalAccessException e) {
+ LOGGER.error("Could not override configuration type on key: {}", key);
+ }
+ }
+
+ return loadConfigurations(cfgPath, distributedModule, includeDefaults);
+ }
+
+ /**
+ * Sets up a configuration registry.
+ *
+ * @param cfgPath Config path.
+ * @param module Module.
+ * @param includeDefaults Include defaults.
+ */
+ private static ConfigurationRegistry loadConfigurations(Path cfgPath, ConfigurationModule module, boolean includeDefaults) {
+ ConfigurationTreeGenerator localConfigurationGenerator =
+ new ConfigurationTreeGenerator(module.rootKeys(), module.schemaExtensions(), module.polymorphicSchemaExtensions());
+
+ LocalFileConfigurationStorage localFileConfigurationStorage = (includeDefaults)
+ ? new LocalFileConfigurationStorage(cfgPath, localConfigurationGenerator, module)
+ : new NoDefaultsStorageConfiguration(cfgPath, localConfigurationGenerator, module);
+
+ // Remove the authentication validator because I cannot get the module.patchConfigurationWithDynamicDefaults(change); to work.
+ // TODO: Check if this will create an error on the service.
+ Set<? extends Validator<?, ?>> myValidators = module.validators()
+ .stream()
+ .filter(v -> !(v instanceof AuthenticationProvidersValidatorImpl))
+ .collect(Collectors.toSet());
+
+ ConfigurationValidator localConfigurationValidator =
+ ConfigurationValidatorImpl.withDefaultValidators(localConfigurationGenerator, myValidators);
+
+ var nodeCfgMgr = new ConfigurationManager(module.rootKeys(), localFileConfigurationStorage, localConfigurationGenerator,
+ localConfigurationValidator, c -> {}, s -> false);
+
+ nodeCfgMgr.startAsync(new ComponentContext()).join();
+ return nodeCfgMgr.configurationRegistry();
+ }
+
+ /**
+ * Loads all the configuration modules in the main classloader.
+ */
+ private static ConfigurationModules loadConfigurationModules() {
+ // TODO: Copied
+ var modulesProvider = new ServiceLoaderModulesProvider();
+ List<ConfigurationModule> modules = modulesProvider.modules(null);
+
+ if (modules.isEmpty()) {
+ throw new IllegalStateException("No configuration modules were loaded, this means Ignite cannot start. "
+ + "Please make sure that the classloader for loading services is correct.");
+ }
+
+ return new ConfigurationModules(modules);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/loader/BeanWrapperSpy.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/loader/BeanWrapperSpy.java
new file mode 100644
index 00000000000..feb1ba0d52d
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/loader/BeanWrapperSpy.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.loader;
+
+import java.beans.PropertyDescriptor;
+import java.beans.PropertyEditor;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.commons.collections4.MultiValuedMap;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.BeanWrapper;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.InvalidPropertyException;
+import org.springframework.beans.PropertyValue;
+import org.springframework.beans.PropertyValues;
+import org.springframework.beans.TypeMismatchException;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.beans.factory.support.RootBeanDefinition;
+import org.springframework.core.MethodParameter;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.core.convert.TypeDescriptor;
+import org.springframework.lang.Nullable;
+
+/**
+ * A custom implementation that tracks which properties where explicitly set in the bean.
+ */
+public class BeanWrapperSpy implements BeanWrapper {
+ private final MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> propertyTracker;
+
+ private final BeanWrapper base;
+
+ private final BeanDefinition mdb;
+
+ /**
+ * Constructor.
+ *
+ * @param propertyTracker Map that tracks
+ * @param base Delegated BeanWrapper
+ * @param mbd Root definition
+ */
+ public BeanWrapperSpy(MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> propertyTracker, BeanWrapper base,
+ RootBeanDefinition mbd) {
+ this.propertyTracker = propertyTracker;
+ this.base = base;
+ this.mdb = mbd;
+ }
+
+ private void registerProperty(PropertyValue pv) {
+ Object instance = getWrappedInstance();
+ PropertyDescriptor descriptor = getPropertyDescriptor(pv.getName());
+ this.propertyTracker.put(instance, Pair.of(descriptor, this.mdb));
+ }
+
+ // Delegated methods.
+
+ @Override
+ public int getAutoGrowCollectionLimit() {
+ return base.getAutoGrowCollectionLimit();
+ }
+
+ @Override
+ public void setAutoGrowCollectionLimit(int autoGrowCollectionLimit) {
+ base.setAutoGrowCollectionLimit(autoGrowCollectionLimit);
+ }
+
+ @Override
+ public Object getWrappedInstance() {
+ return base.getWrappedInstance();
+ }
+
+ @Override
+ public Class<?> getWrappedClass() {
+ return base.getWrappedClass();
+ }
+
+ @Override
+ public PropertyDescriptor[] getPropertyDescriptors() {
+ return base.getPropertyDescriptors();
+ }
+
+ @Override
+ public PropertyDescriptor getPropertyDescriptor(String propertyName) throws InvalidPropertyException {
+ return base.getPropertyDescriptor(propertyName);
+ }
+
+ @Override
+ @Nullable
+ public ConversionService getConversionService() {
+ return base.getConversionService();
+ }
+
+ @Override
+ public void setConversionService(ConversionService conversionService) {
+ base.setConversionService(conversionService);
+ }
+
+ @Override
+ public boolean isExtractOldValueForEditor() {
+ return base.isExtractOldValueForEditor();
+ }
+
+ @Override
+ public void setExtractOldValueForEditor(boolean extractOldValueForEditor) {
+ base.setExtractOldValueForEditor(extractOldValueForEditor);
+ }
+
+ @Override
+ public boolean isAutoGrowNestedPaths() {
+ return base.isAutoGrowNestedPaths();
+ }
+
+ @Override
+ public void setAutoGrowNestedPaths(boolean autoGrowNestedPaths) {
+ base.setAutoGrowNestedPaths(autoGrowNestedPaths);
+ }
+
+ @Override
+ public boolean isReadableProperty(String propertyName) {
+ return base.isReadableProperty(propertyName);
+ }
+
+ @Override
+ public boolean isWritableProperty(String propertyName) {
+ return base.isWritableProperty(propertyName);
+ }
+
+ @Override
+ @Nullable
+ public Class<?> getPropertyType(String propertyName) throws BeansException {
+ return base.getPropertyType(propertyName);
+ }
+
+ @Override
+ @Nullable
+ public TypeDescriptor getPropertyTypeDescriptor(String propertyName) throws BeansException {
+ return base.getPropertyTypeDescriptor(propertyName);
+ }
+
+ @Override
+ @Nullable
+ public Object getPropertyValue(String propertyName) throws BeansException {
+ return base.getPropertyValue(propertyName);
+ }
+
+ @Override
+ public void setPropertyValue(String propertyName, Object value) throws BeansException {
+ base.setPropertyValue(propertyName, value);
+ }
+
+ @Override
+ public void setPropertyValue(PropertyValue pv) throws BeansException {
+ base.setPropertyValue(pv);
+ registerProperty(pv);
+ }
+
+ @Override
+ public void setPropertyValues(Map<?, ?> map) throws BeansException {
+ base.setPropertyValues(map);
+ }
+
+ @Override
+ public void setPropertyValues(PropertyValues pvs) throws BeansException {
+ base.setPropertyValues(pvs);
+ for (PropertyValue pv : pvs) {
+ registerProperty(pv);
+ }
+ }
+
+ @Override
+ public void setPropertyValues(PropertyValues pvs, boolean ignoreUnknown) throws BeansException {
+ base.setPropertyValues(pvs, ignoreUnknown);
+ }
+
+ @Override
+ public void setPropertyValues(PropertyValues pvs, boolean ignoreUnknown, boolean ignoreInvalid) throws BeansException {
+ base.setPropertyValues(pvs, ignoreUnknown, ignoreInvalid);
+ }
+
+ @Override
+ public void registerCustomEditor(Class<?> requiredType, PropertyEditor propertyEditor) {
+ base.registerCustomEditor(requiredType, propertyEditor);
+ }
+
+ @Override
+ public void registerCustomEditor(Class<?> requiredType, String propertyPath, PropertyEditor propertyEditor) {
+ base.registerCustomEditor(requiredType, propertyPath, propertyEditor);
+ }
+
+ @Override
+ @Nullable
+ public PropertyEditor findCustomEditor(Class<?> requiredType, String propertyPath) {
+ return base.findCustomEditor(requiredType, propertyPath);
+ }
+
+ @Override
+ @Nullable
+ public <T> T convertIfNecessary(Object value, Class<T> requiredType) throws TypeMismatchException {
+ return base.convertIfNecessary(value, requiredType);
+ }
+
+ @Override
+ @Nullable
+ public <T> T convertIfNecessary(Object value, Class<T> requiredType, MethodParameter methodParam) throws TypeMismatchException {
+ // This is some crazy stuff.
+ // Since we allow nulls on beans that we cannot create, we might end up with nulls in primitive collections.
+ // The solution is to strip them from the source collection.
+ // TODO: This should not have been here. But I didn't want to create yet another decorator.
+ Object newValue = value;
+ if (requiredType.isArray() && requiredType.getComponentType().isPrimitive()) {
+ if (value instanceof Collection) {
+ newValue = ((Collection<?>) value).stream().filter(Objects::nonNull).toArray();
+ } else if (value.getClass().isArray() && !value.getClass().isPrimitive()) {
+ newValue = Arrays.stream((Object[]) value).filter(Objects::nonNull).toArray();
+ }
+ }
+
+ return base.convertIfNecessary(newValue, requiredType, methodParam);
+ }
+
+ @Override
+ @Nullable
+ public <T> T convertIfNecessary(Object value, Class<T> requiredType, Field field) throws TypeMismatchException {
+ return base.convertIfNecessary(value, requiredType, field);
+ }
+
+ @Override
+ @Nullable
+ public <T> T convertIfNecessary(Object value, Class<T> requiredType, TypeDescriptor typeDescriptor) throws TypeMismatchException {
+ return base.convertIfNecessary(value, requiredType, typeDescriptor);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/loader/CustomBeanFactory.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/loader/CustomBeanFactory.java
new file mode 100644
index 00000000000..419d1a8f882
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/loader/CustomBeanFactory.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.loader;
+
+import java.beans.PropertyDescriptor;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import org.apache.commons.collections4.MultiValuedMap;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanWrapper;
+import org.springframework.beans.factory.BeanCreationException;
+import org.springframework.beans.factory.CannotLoadBeanClassException;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.beans.factory.support.DefaultListableBeanFactory;
+import org.springframework.beans.factory.support.RootBeanDefinition;
+
+/**
+ * This factory overrides the BeanWrapper creation.
+ */
+public class CustomBeanFactory extends DefaultListableBeanFactory {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(CustomBeanFactory.class);
+
+ private static final Constructor<?> NULL_BEAN_CONSTRUCTOR;
+
+ static {
+ try {
+ Class<?> nullBeanClass = Class.forName("org.springframework.beans.factory.support.NullBean");
+ NULL_BEAN_CONSTRUCTOR = nullBeanClass.getDeclaredConstructor();
+ NULL_BEAN_CONSTRUCTOR.setAccessible(true);
+ } catch (ClassNotFoundException | NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private final MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> propertyTracker;
+
+ public CustomBeanFactory(MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> tracker) {
+ this.propertyTracker = tracker;
+ }
+
+ @Override
+ protected BeanWrapper instantiateBean(String beanName, RootBeanDefinition mbd) {
+ BeanWrapper wrapper = super.instantiateBean(beanName, mbd);
+ // Might be nullable
+ return (wrapper != null) ? new BeanWrapperSpy(this.propertyTracker, wrapper, mbd) : null;
+ }
+
+ @Override
+ protected Class<?> determineTargetType(String beanName, RootBeanDefinition mbd, Class<?>... typesToMatch) {
+ try {
+ return super.determineTargetType(beanName, mbd, typesToMatch);
+ } catch (CannotLoadBeanClassException ex) {
+ // This should be effectively NullBean, but it is package private.
+ LOGGER.warn("Could determineTargetType for bean {}; skipping...", beanName, ex);
+ return null;
+ }
+ }
+
+ @Override
+ protected Object createBean(String beanName, RootBeanDefinition mbd, Object[] args) throws BeanCreationException {
+ // This will make sure that we ship components that are not in the classPath
+ try {
+ return super.createBean(beanName, mbd, args);
+ } catch (CannotLoadBeanClassException | BeanCreationException ex) {
+ LOGGER.warn("Could not create bean {}; skipping...", beanName, ex);
+ try {
+ return NULL_BEAN_CONSTRUCTOR.newInstance();
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
+ // this can still allow the program to load
+ LOGGER.error("Could not create NULL BEAN", e);
+ return null;
+ }
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/registry/CombinedConfigRegistry.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/registry/CombinedConfigRegistry.java
new file mode 100644
index 00000000000..2d44fe0021f
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/registry/CombinedConfigRegistry.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.registry;
+
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite3.configuration.ConfigurationTree;
+import org.apache.ignite3.configuration.RootKey;
+import org.apache.ignite3.internal.configuration.ConfigurationRegistry;
+import org.apache.ignite3.internal.manager.ComponentContext;
+import org.apache.ignite3.internal.util.IgniteUtils;
+
+/**
+ * Registry has two separated internal registries for cluster and node configurations.
+ */
+public class CombinedConfigRegistry implements ConfigurationRegistryInterface {
+
+ private final ConfigurationRegistry localRegistry;
+
+ private final ConfigurationRegistry distributedRegistry;
+
+ public CombinedConfigRegistry(ConfigurationRegistry localRegistry, ConfigurationRegistry distributedRegistry) {
+ this.localRegistry = localRegistry;
+ this.distributedRegistry = distributedRegistry;
+ }
+
+ @Override
+ public CompletableFuture<Void> startAsync(ComponentContext context) {
+ return IgniteUtils.startAsync(context, localRegistry, distributedRegistry);
+ }
+
+ @Override
+ public CompletableFuture<Void> stopAsync(ComponentContext context) {
+ return IgniteUtils.stopAsync(context, localRegistry, distributedRegistry);
+ }
+
+ @Override public <V, C, T extends ConfigurationTree<V, C>> T getConfiguration(RootKey<T, V> rootKey) {
+ return Optional.ofNullable(localRegistry.getConfiguration(rootKey))
+ .orElseGet(() -> distributedRegistry.getConfiguration(rootKey));
+ }
+
+ public CompletableFuture<Void> onDefaultsPersisted() {
+ return CompletableFuture.allOf(localRegistry.onDefaultsPersisted(), distributedRegistry.onDefaultsPersisted());
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/registry/ConfigurationRegistryInterface.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/registry/ConfigurationRegistryInterface.java
new file mode 100644
index 00000000000..6c6e33c369c
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/registry/ConfigurationRegistryInterface.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.registry;
+
+import org.apache.ignite3.configuration.ConfigurationTree;
+import org.apache.ignite3.configuration.RootKey;
+import org.apache.ignite3.internal.manager.IgniteComponent;
+
+/**
+ * Custom Configuration Registry Interface.
+ */
+public interface ConfigurationRegistryInterface extends IgniteComponent {
+ <V, C, T extends ConfigurationTree<V, C>> T getConfiguration(RootKey<T, V> rootKey);
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/storage/NoDefaultsStorageConfiguration.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/storage/NoDefaultsStorageConfiguration.java
new file mode 100644
index 00000000000..02f7cbea397
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/config/storage/NoDefaultsStorageConfiguration.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.storage;
+
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite3.configuration.ConfigurationModule;
+import org.apache.ignite3.internal.configuration.ConfigurationTreeGenerator;
+import org.apache.ignite3.internal.configuration.storage.ConfigurationStorageListener;
+import org.apache.ignite3.internal.configuration.storage.Data;
+import org.apache.ignite3.internal.configuration.storage.LocalFileConfigurationStorage;
+
+/**
+ * Patched version of the LocalFileConfiguration Storage that does not store defaults.
+ * This class should be considered experimental. The best solution would be to skip the usage of the
+ * Configuration registry entirely because we do not use it's features.
+ */
+public class NoDefaultsStorageConfiguration extends LocalFileConfigurationStorage {
+ // TODO: Check if it needs to be volatile. The listener must see the defaults after .write().
+ private Map<String, ? extends Serializable> defaultValues = null;
+
+ public NoDefaultsStorageConfiguration(Path configPath, ConfigurationTreeGenerator generator, ConfigurationModule module) {
+ super(configPath, generator, module);
+ }
+
+ @Override
+ public CompletableFuture<Data> readDataOnRecovery() {
+ return CompletableFuture.completedFuture(new Data(Collections.emptyMap(), 0));
+ }
+
+ @Override public void registerConfigurationListener(ConfigurationStorageListener lsnr) {
+ // Decorate the listener to let the user know we handled the defaults properly.
+ super.registerConfigurationListener(data -> {
+ // Check if this is the default configs
+ if (data.changeId() == 1L && data.values().isEmpty()) {
+ // Return the default values to the listener.
+ return lsnr.onEntriesChanged(new Data(defaultValues, data.changeId()));
+ } else {
+ return lsnr.onEntriesChanged(data);
+ }
+ });
+ }
+
+ @Override public CompletableFuture<Boolean> write(Map<String, ? extends Serializable> newValues, long ver) {
+ // Skipping default storage. It assumes that the defaults will be stored in version 0.
+ if (ver == 0) {
+ // The defaults are stored for later use.
+ this.defaultValues = newValues;
+ return super.write(Collections.emptyMap(), ver);
+ } else {
+ return super.write(newValues, ver);
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/sql/FieldNameConflictException.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/sql/FieldNameConflictException.java
new file mode 100644
index 00000000000..44cc9ab0102
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/sql/FieldNameConflictException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.sql;
+
+import java.util.Set;
+
+/** Defines a situation when it was not possible to assign a field name due to a conflict with other fields. */
+public class FieldNameConflictException extends RuntimeException {
+
+ private FieldNameConflictException(String message) {
+ super(message);
+ }
+
+ public FieldNameConflictException(SqlDdlGenerator.InspectedField inspectedField, Set<String> knownFieldNames) {
+ super("Duplicated field name for:" + inspectedField + " in " + knownFieldNames);
+ }
+
+ public static FieldNameConflictException forSpecificField(String fieldName, String expectedValue, String foundValue) {
+ return new FieldNameConflictException(
+ String.format("Unexpected type for field: %s - Expected:%s Found:%s", fieldName, expectedValue, foundValue));
+ }
+
+ public static FieldNameConflictException forSpecificField(String label, String fieldName) {
+ return new FieldNameConflictException(String.format("Unexpected null type for %s field with name: %s", label, fieldName));
+ }
+
+ /**
+ * Creates an exception for an Unknown conflict type.
+ *
+ * @param fieldName Field name.
+ * @param fieldType Field type.
+ * @return The created exception.
+ */
+ public static FieldNameConflictException forUnknownType(String fieldName, String fieldType) {
+ return new FieldNameConflictException(
+ String.format("Unknown non-native type for field: %s - %s. Consider adding this type to the classpath.",
+ fieldName, fieldType));
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/sql/SqlDdlGenerator.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/sql/SqlDdlGenerator.java
new file mode 100644
index 00000000000..48d42ff970c
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/sql/SqlDdlGenerator.java
@@ -0,0 +1,841 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.sql;
+
+import static java.util.function.Predicate.not;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Modifier;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.migrationtools.tablemanagement.TableTypeRegistry;
+import org.apache.ignite.migrationtools.tablemanagement.TableTypeRegistryMapImpl;
+import org.apache.ignite.migrationtools.utils.ClassnameUtils;
+import org.apache.ignite3.catalog.ColumnType;
+import org.apache.ignite3.catalog.definitions.ColumnDefinition;
+import org.apache.ignite3.catalog.definitions.TableDefinition;
+import org.apache.ignite3.internal.catalog.sql.CatalogExtensions;
+import org.apache.ignite3.table.mapper.Mapper;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Generates a SQL DDL Script from Ignite 2 Cache Configurations and custom type hints.
+ */
+public class SqlDdlGenerator {
+ /** Name of the column that will be used to store fields that cannot be serialized natively, like nested objects in POJOs. */
+ public static final String EXTRA_FIELDS_COLUMN_NAME = "__EXTRA__";
+
+ private static final String[] VALUE_FIELD_NAME_CANDIDATES = new String[] {"VAL", "VAL_OBJ", "TARGET_OBJ"};
+
+ private static final String[] ID_FIELD_NAME_CANDIDATES = new String[] {"ID", "KEY"};
+
+ // Extracted from org.apache.ignite.internal.sql.engine.sql.SqlReservedWordsTest#RESERVED_WORDS
+
+ /** List of keywords reserved in Ignite SQL. */
+ private static final Set<String> RESERVED_WORDS = Set.of(
+ "ABS",
+ "ALL", // UNION ALL
+ "ALTER",
+ "AND",
+ "ANY",
+ "ARRAY",
+ "ARRAY_MAX_CARDINALITY",
+ "AS",
+ "ASYMMETRIC", // BETWEEN ASYMMETRIC .. AND ..
+ "AVG",
+ "BETWEEN",
+ "BOTH", // TRIM(BOTH .. FROM ..)
+ "BY", // GROUP BY
+ "CACHE",
+ "CALL",
+ "CARDINALITY",
+ "CASE",
+ "CAST",
+ "CEILING",
+ "CHAR",
+ "CHARACTER",
+ "CHARACTER_LENGTH",
+ "CHAR_LENGTH",
+ "COALESCE",
+ "COLLECT",
+ "COLUMN",
+ "CONSTRAINT",
+ "CONVERT",
+ "COUNT",
+ "COVAR_POP",
+ "COVAR_SAMP",
+ "CREATE",
+ "CROSS", // CROSS JOIN
+ "CUBE",
+ "CUME_DIST",
+ "CURRENT",
+ "CURRENT_CATALOG",
+ "CURRENT_DATE",
+ "CURRENT_DEFAULT_TRANSFORM_GROUP",
+ "CURRENT_PATH",
+ "CURRENT_ROLE",
+ "CURRENT_ROW",
+ "CURRENT_SCHEMA",
+ "CURRENT_TIME",
+ "CURRENT_TIMESTAMP",
+ "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
+ "CURRENT_USER",
+ "DATE",
+ "DATETIME",
+ "DECIMAL",
+ "DEFAULT",
+ "DELETE",
+ "DENSE_RANK",
+ "DESCRIBE",
+ "DISTINCT",
+ "DROP",
+ "ELEMENT",
+ "ELSE",
+ "EVERY",
+ "EXCEPT",
+ "EXISTS",
+ "EXP",
+ "EXPLAIN",
+ "EXTEND",
+ "EXTRACT",
+ "FALSE",
+ "FETCH",
+ "FILTER",
+ "FIRST_VALUE",
+ "FLOOR",
+ "FOR", // SUBSTRING(.. FROM .. FOR ..)
+ "FRIDAY",
+ "FROM",
+ "FULL", // FULL JOIN
+ "FUSION",
+ "GRANTS",
+ "GROUP",
+ "GROUPING",
+ "HAVING",
+ "HOUR",
+ "IDENTIFIED",
+ "IF",
+ "IN",
+ "INDEX",
+ "INNER",
+ "INSERT",
+ "INTERSECT",
+ "INTERSECTION",
+ "INTERVAL",
+ "INTO",
+ "IS",
+ "JOIN",
+ "JSON_SCOPE",
+ "LAG",
+ "LAST_VALUE",
+ "LEAD",
+ "LEADING", // TRIM(LEADING .. FROM ..)
+ "LEFT", // LEFT JOIN
+ "LIKE",
+ "LIMIT",
+ "LN",
+ "LOCALTIME",
+ "LOCALTIMESTAMP",
+ "LOWER",
+ "MATCH_RECOGNIZE",
+ "MAX",
+ "MERGE",
+ "MIN",
+ "MINUS",
+ "MINUTE",
+ "MOD",
+ "MONDAY",
+ "MONTH",
+ "MULTISET",
+ "NATURAL", // NATURAL JOIN
+ "NEW",
+ "NEXT",
+ "NOT",
+ "NTH_VALUE",
+ "NTILE",
+ "NULL",
+ "NULLIF",
+ "OCTET_LENGTH",
+ "OFFSET",
+ "ON",
+ "OR",
+ "ORDER",
+ "OUTER", // OUTER JOIN
+ "OVER",
+ "PARTITION",
+ "PERCENTILE_CONT",
+ "PERCENTILE_DISC",
+ "PERCENT_RANK",
+ "PERIOD",
+ "PERMUTE",
+ "POWER",
+ "PRECISION",
+ "PRIMARY",
+ "QUALIFY",
+ "RANK",
+ "REGR_COUNT",
+ "REGR_SXX",
+ "REGR_SYY",
+ "RENAME",
+ "RESET",
+ "RIGHT",
+ "ROLES",
+ "ROLLUP",
+ "ROW",
+ "ROW_NUMBER",
+ "SATURDAY",
+ "SECOND",
+ "SELECT",
+ "SESSION_USER",
+ "SET",
+ "SOME",
+ "SPECIFIC",
+ "SQRT",
+ "STDDEV_POP",
+ "STDDEV_SAMP",
+ "STREAM",
+ "SUBSTRING",
+ "SUM",
+ "SUNDAY",
+ "SYMMETRIC", // BETWEEN SYMMETRIC .. AND ..
+ "SYSTEM_TIME",
+ "SYSTEM_USER",
+ "TABLE",
+ "TABLESAMPLE",
+ "THEN",
+ "THURSDAY",
+ "TIME",
+ "TIMESTAMP",
+ "TO",
+ "TRAILING", // TRIM(TRAILING .. FROM ..)
+ "TRUE",
+ "TRUNCATE",
+ "TUESDAY",
+ "UESCAPE",
+ "UNION",
+ "UNKNOWN",
+ "UPDATE",
+ "UPPER",
+ "UPSERT",
+ "USER",
+ "USERS",
+ "USING",
+ "VALUE",
+ "VALUES",
+ "VAR_POP",
+ "VAR_SAMP",
+ "WEDNESDAY",
+ "WHEN",
+ "WHERE",
+ "WINDOW",
+ "WITH",
+ "WITHIN",
+ "YEAR"
+ );
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SqlDdlGenerator.class);
+
+ private static final String DEFAULT_SCHEMA = "PUBLIC";
+
+ private static final int DEFAULT_BINARY_FIELD_LENGTH = 1024;
+
+ private static final Map<Class<?>, ColumnType<?>> COL_TYPE_REF;
+
+ static {
+ try {
+ COL_TYPE_REF = (Map<Class<?>, ColumnType<?>>) FieldUtils.readDeclaredStaticField(ColumnType.class, "TYPES", true);
+ // TODO: IGNITE-23268 Remove
+ COL_TYPE_REF.remove(java.util.Date.class);
+
+ var constructor = ColumnType.class.getDeclaredConstructor(Class.class, String.class);
+ constructor.setAccessible(true);
+ constructor.newInstance(Character.class, "CHAR");
+ constructor.newInstance(BitSet.class, "VARBINARY");
+ constructor.newInstance(LocalTime.class, "TIME");
+ constructor.newInstance(LocalDate.class, "DATE");
+ constructor.newInstance(LocalDateTime.class, "TIMESTAMP");
+ constructor.newInstance(Instant.class, "TIMESTAMP");
+ constructor.newInstance(java.util.Date.class, "TIMESTAMP");
+ constructor.newInstance(Enum.class, "VARCHAR");
+ // TODO: IGNITE-23268 Remove
+ constructor.newInstance(java.sql.Date.class, "DATE");
+ constructor.newInstance(java.sql.Time.class, "TIME");
+ constructor.newInstance(java.sql.Timestamp.class, "TIMESTAMP");
+ // Collections
+ constructor.newInstance(Collection.class, "VARBINARY");
+ constructor.newInstance(List.class, "VARBINARY");
+ constructor.newInstance(Set.class, "VARBINARY");
+ // Primitive Arrays
+ constructor.newInstance(boolean[].class, "VARBINARY");
+ constructor.newInstance(char[].class, "VARBINARY");
+ constructor.newInstance(short[].class, "VARBINARY");
+ constructor.newInstance(int[].class, "VARBINARY");
+ constructor.newInstance(long[].class, "VARBINARY");
+ constructor.newInstance(float[].class, "VARBINARY");
+ constructor.newInstance(double[].class, "VARBINARY");
+ constructor.newInstance(String[].class, "VARBINARY");
+ } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException | InstantiationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private final ClassLoader clientClassLoader;
+
+ private final TableTypeRegistry tableTypeRegistry;
+
+ private final boolean allowExtraFields;
+
+ public SqlDdlGenerator() {
+ this(new TableTypeRegistryMapImpl());
+ }
+
+ public SqlDdlGenerator(TableTypeRegistry tableTypeRegistry) {
+ this(tableTypeRegistry, false);
+ }
+
+ public SqlDdlGenerator(TableTypeRegistry tableTypeRegistry, boolean allowExtraFields) {
+ this(SqlDdlGenerator.class.getClassLoader(), tableTypeRegistry, allowExtraFields);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param clientClassLoader Custom classloader to use.
+ * @param tableTypeRegistry Table type registry implementation to use.
+ * @param allowExtraFields If true, the extra fields column will be added to generated tables.
+ */
+ public SqlDdlGenerator(ClassLoader clientClassLoader, TableTypeRegistry tableTypeRegistry, boolean allowExtraFields) {
+ this.clientClassLoader = clientClassLoader;
+ this.tableTypeRegistry = tableTypeRegistry;
+ this.allowExtraFields = allowExtraFields;
+ }
+
+ private static boolean isPrimitiveType(Class<?> type) {
+ return type.isEnum() || Mapper.nativelySupported(type) || COL_TYPE_REF.containsKey(type);
+ }
+
+ private static List<InspectedField> inspectType(Class<?> type) {
+ Class<?> rootType = ClassUtils.primitiveToWrapper(type);
+ String rootTypeName = rootType.getName();
+
+ if (rootType.isArray() || Collection.class.isAssignableFrom(rootType)) {
+ return Collections.singletonList(new InspectedField(null, rootTypeName, FieldType.ARRAY));
+ } else if (isPrimitiveType(rootType)) {
+ return Collections.singletonList(new InspectedField(null, rootTypeName, FieldType.PRIMITIVE));
+ } else {
+ Field[] fields = rootType.getDeclaredFields();
+ List<InspectedField> ret = new ArrayList<>(fields.length);
+ for (Field field : fields) {
+ Class<?> origFieldType = field.getType();
+ Class<?> wrappedFieldType = ClassUtils.primitiveToWrapper(origFieldType);
+ if (shouldPersistField(field) && isPrimitiveType(wrappedFieldType)) {
+ boolean nullable = !origFieldType.isPrimitive();
+ ret.add(new InspectedField(field.getName(), wrappedFieldType.getName(), FieldType.POJO_ATTRIBUTE, nullable));
+ }
+ }
+
+ return ret;
+ }
+ }
+
+ private static void addToQueryEntity(QueryEntity qe, InspectedField inspectedField, boolean nullable) {
+ // Set precision
+ if (inspectedField.fieldType == FieldType.ARRAY) {
+ qe.getFieldsPrecision().putIfAbsent(inspectedField.fieldName, DEFAULT_BINARY_FIELD_LENGTH);
+ }
+
+ if (!nullable) {
+ qe.getNotNullFields().add(inspectedField.fieldName);
+ }
+
+ qe.getFields().put(inspectedField.fieldName, inspectedField.typeName);
+ }
+
+ private static void addToQueryEntity(QueryEntity qe, InspectedField inspectedField) {
+ addToQueryEntity(qe, inspectedField, inspectedField.isNullable());
+ }
+
+ private static boolean shouldPersistField(Field field) {
+ var mods = field.getModifiers();
+ return !Modifier.isStatic(mods) && !Modifier.isTransient(mods);
+ }
+
+ private static String sanitizeColumnName(String columnName) {
+ // If is quoted the field is accepted as is.
+ if (columnName.startsWith("\"")) {
+ return columnName;
+ }
+
+ String uppercaseColumnName = columnName.toUpperCase();
+ // If the column name is reserved then we must quote it.
+ return (RESERVED_WORDS.contains(uppercaseColumnName)) ? '"' + uppercaseColumnName + '"' : columnName;
+ }
+
+ public static String createDdlQuery(TableDefinition def) {
+ return createDdlQuery(Collections.singletonList(def));
+ }
+
+ /**
+ * Create SQL DDL Statements from the provided list of table definitions.
+ *
+ * @param defs Table definitions.
+ * @return String composed by the SQL DDL Statements.
+ */
+ public static String createDdlQuery(List<TableDefinition> defs) {
+ return defs.stream()
+ .map(CatalogExtensions::sqlFromTableDefinition)
+ .collect(Collectors.joining("\n\n"));
+ }
+
+ @Nullable
+ private static Map.Entry<String, String> keyField(QueryEntity qe) {
+ String fname = qe.getKeyFieldName();
+ return (fname != null) ? Map.entry(fname, qe.getKeyType()) : null;
+ }
+
+ @Nullable
+ private static Map.Entry<String, String> valField(QueryEntity qe) {
+ String fname = qe.getValueFieldName();
+ return (fname != null) ? Map.entry(fname, qe.getValueType()) : null;
+ }
+
+ private List<InspectedField> inspectTypeName(String typeName, String typeDescr) {
+ try {
+ Class<?> type = ClassUtils.getClass(this.clientClassLoader, typeName);
+ return inspectType(type);
+ } catch (ClassNotFoundException e) {
+ LOGGER.warn("Could not find {} class to enrich the QueryEntity: {}", typeDescr, typeName);
+ return Collections.emptyList();
+ }
+ }
+
+ private QueryEntity populateQueryEntity(QueryEntity qe, boolean allowExtraFields) throws FieldNameConflictException {
+ // Make sure QE has non-null maps
+ {
+ if (qe.getNotNullFields() == null) {
+ qe.setNotNullFields(new HashSet<>());
+ }
+
+ if (qe.getKeyFields() == null) {
+ qe.setKeyFields(new HashSet<>());
+ }
+
+ if (qe.getFieldsPrecision() == null) {
+ qe.setFieldsPrecision(new HashMap<>());
+ }
+ }
+
+ // Go over existing fields in the QE to they are correct and there are not silly nulls.
+ {
+ // AI2 may define primitive field types, however, they still mark them as nullable somehow.
+ for (var e : qe.getFields().entrySet()) {
+ e.setValue(ClassnameUtils.ensureWrapper(e.getValue()));
+ }
+
+ // Make sure keyFieldName and keyFieldTypes are set.
+ List<Map.Entry<String, String>> keyFields =
+ qe.getFields().entrySet().stream().filter(f -> qe.getKeyFields().contains(f.getKey())).collect(Collectors.toList());
+ @Nullable String keyFieldName = qe.getKeyFieldName();
+ if (keyFieldName != null) {
+ @Nullable String foundType = qe.findKeyType();
+ if (foundType == null) {
+ throw FieldNameConflictException.forSpecificField("key", keyFieldName);
+ }
+
+ qe.getFields().put(keyFieldName, ClassnameUtils.ensureWrapper(foundType));
+ qe.getKeyFields().add(keyFieldName);
+ } else if (keyFields.size() == 1) {
+ qe.setKeyFieldName(keyFields.get(0).getKey());
+ qe.setKeyType(keyFields.get(0).getValue());
+ }
+
+ // Make sure valFieldName and valFieldTypes are set.
+ List<Map.Entry<String, String>> valueFields =
+ qe.getFields().entrySet().stream().filter(f -> !qe.getKeyFields().contains(f.getKey())).collect(Collectors.toList());
+ @Nullable String valFieldName = qe.getValueFieldName();
+ if (valFieldName != null) {
+ @Nullable String foundType = qe.findValueType();
+ if (foundType == null) {
+ throw FieldNameConflictException.forSpecificField("value", valFieldName);
+ }
+
+ qe.getFields().put(valFieldName, ClassnameUtils.ensureWrapper(foundType));
+ } else if (valueFields.size() == 1) {
+ qe.setValueFieldName(valueFields.get(0).getKey());
+ qe.setValueType(valueFields.get(0).getValue());
+ }
+
+ // Mark keyFields as not nullable.
+ qe.getNotNullFields().addAll(qe.getKeyFields());
+ }
+
+ // Inspect classes that are on the classpath
+ List<InspectedField> keyFields = inspectTypeName(qe.getKeyType(), "KEY");
+ List<InspectedField> valFields = inspectTypeName(qe.getValueType(), "VALUE");
+
+ // Check duplicated field names, and assign custom field names
+ {
+ Set<String> fieldNames = new HashSet<>(qe.getFields().size() + keyFields.size() + valFields.size());
+ List<Map.Entry<InspectedField, Supplier<String>>> unnamedFields = new ArrayList<>(keyFields.size() + valFields.size());
+
+ Supplier<String> keyFieldNameCandidates = new FieldNameCandidateSupplier(ID_FIELD_NAME_CANDIDATES, n -> "KEY_" + n);
+ Supplier<String> valFieldCandidates = new FieldNameCandidateSupplier(VALUE_FIELD_NAME_CANDIDATES, n -> "VAL_" + n);
+ Stream<Triple<InspectedField, Supplier<Map.Entry<String, String>>, Supplier<String>>> x = Stream.concat(
+ keyFields.stream().map(f -> Triple.of(f, () -> keyField(qe), keyFieldNameCandidates)),
+ valFields.stream().map(f -> Triple.of(f, () -> valField(qe), valFieldCandidates))
+ );
+
+ // Add fields already in the QE
+ // Also add the aliases, we don't need collisions on that either.
+ Stream.concat(qe.getFields().keySet().stream(), qe.getAliases().values().stream())
+ .map(String::toUpperCase).forEach(fieldNames::add);
+
+ for (Iterator<Triple<InspectedField, Supplier<Map.Entry<String, String>>, Supplier<String>>> it = x.iterator();
+ it.hasNext(); ) {
+ Triple<InspectedField, Supplier<Map.Entry<String, String>>, Supplier<String>> entry = it.next();
+ InspectedField inspectedField = entry.getLeft();
+
+ if (inspectedField.fieldName != null) {
+ String fieldNameUpperCase = inspectedField.fieldName.toUpperCase();
+ if (!fieldNames.contains(fieldNameUpperCase)) {
+ fieldNames.add(fieldNameUpperCase);
+ } else {
+ // I've seen some weird cases where there was case mismatch between the class attr name and the qe field.
+ // To accept as the same field, both the name (without casing) and the field type must match.
+ Optional<Map.Entry<String, String>> existingEntryForField = qe.getFields().entrySet().stream()
+ .filter(e -> e.getKey().equalsIgnoreCase(inspectedField.fieldName)
+ && e.getValue().equals(inspectedField.typeName))
+ .findFirst();
+
+ if (existingEntryForField.isPresent()) {
+ // We will switch our inspected field name to match the casing in the QE and hope for the best.
+ inspectedField.fieldName = existingEntryForField.get().getKey();
+ } else {
+ throw new FieldNameConflictException(inspectedField, fieldNames);
+ }
+ }
+ } else if (inspectedField.fieldType == FieldType.PRIMITIVE) {
+ @Nullable var field = entry.getMiddle().get();
+ if (field == null) {
+ unnamedFields.add(Map.entry(inspectedField, entry.getRight()));
+ } else if (inspectedField.typeName.equals(field.getValue())) {
+ inspectedField.fieldName = field.getKey();
+ } else {
+ throw FieldNameConflictException.forSpecificField(inspectedField.fieldName, inspectedField.typeName,
+ field.getValue());
+ }
+ } else {
+ unnamedFields.add(Map.entry(inspectedField, entry.getRight()));
+ }
+ }
+
+ // Assign custom field names
+ for (Map.Entry<InspectedField, Supplier<String>> unnamedFieldEntry : unnamedFields) {
+ // Get a valid candidate for the field.
+ String fieldName = Stream.generate(unnamedFieldEntry.getValue())
+ .filter(not(fieldNames::contains))
+ .findFirst()
+ .get();
+
+ fieldNames.add(fieldName);
+ unnamedFieldEntry.getKey().fieldName = fieldName;
+ }
+ }
+
+ // Empty field lists means that the class for the type is not available on the classpath so it must be a pojo.
+ boolean mapsPojo = keyFields.isEmpty() || valFields.isEmpty();
+
+ // Process key fields
+ {
+ // Set keyFieldName if there is only one key field.
+ if (keyFields.size() == 1) {
+ qe.setKeyFieldName(keyFields.get(0).fieldName);
+ qe.setKeyType(keyFields.get(0).typeName);
+ }
+
+ for (InspectedField inspectedField : keyFields) {
+ qe.getKeyFields().add(inspectedField.fieldName);
+
+ addToQueryEntity(qe, inspectedField, false);
+ mapsPojo = mapsPojo || inspectedField.fieldType == FieldType.POJO_ATTRIBUTE;
+ }
+ }
+
+ // Process value fields
+ {
+ if (valFields.size() == 1) {
+ qe.setValueFieldName(valFields.get(0).fieldName);
+ qe.setValueType(valFields.get(0).typeName);
+ }
+
+ for (InspectedField inspectedField : valFields) {
+ addToQueryEntity(qe, inspectedField);
+ mapsPojo = mapsPojo || inspectedField.fieldType == FieldType.POJO_ATTRIBUTE;
+ }
+ }
+
+ if (mapsPojo && allowExtraFields) {
+ // TODO: GG-40813 Use a default field value instead of nullable.
+ qe.getFieldsPrecision().putIfAbsent(EXTRA_FIELDS_COLUMN_NAME, DEFAULT_BINARY_FIELD_LENGTH);
+ qe.getFields().put(EXTRA_FIELDS_COLUMN_NAME, byte[].class.getName());
+ }
+
+ return qe;
+ }
+
+ /**
+ * Generate table based on the provided {@link CacheConfiguration}.
+ *
+ * @param cacheCfg The cache configuration.
+ * @return The generate table result.
+ * @throws FieldNameConflictException in case of conflicts during the mapping.
+ */
+ public GenerateTableResult generate(CacheConfiguration<?, ?> cacheCfg) throws FieldNameConflictException {
+ String schema = Optional.ofNullable(cacheCfg.getSqlSchema()).orElse(DEFAULT_SCHEMA);
+ String tableName = "\"" + cacheCfg.getName() + "\"";
+ // TODO: check if tableName needs quoting.
+
+ QueryEntity qryEntity = getOrCreateQueryEntity(cacheCfg);
+
+ int defIdx = 0;
+ int pkIdx = 0;
+ String[] pkColumnNames = new String[qryEntity.getKeyFields().size()];
+ ColumnDefinition[] colDefinitions = new ColumnDefinition[qryEntity.getFields().size()];
+ Map<String, String> fieldNameForColumnMappings = new HashMap<>(qryEntity.getFields().size());
+ for (Map.Entry<String, String> entry : qryEntity.getFields().entrySet()) {
+ String fieldName = entry.getKey();
+ Class<?> klass;
+ try {
+ klass = ClassUtils.getClass(this.clientClassLoader, entry.getValue());
+ if (klass.isEnum()) {
+ klass = Enum.class;
+ }
+ } catch (ClassNotFoundException e) {
+ throw FieldNameConflictException.forUnknownType(fieldName, entry.getValue());
+ }
+
+ Integer precision = qryEntity.getFieldsPrecision().get(fieldName);
+ Integer scale = qryEntity.getFieldsScale().get(fieldName);
+ var colType = ColumnType.of(klass)
+ .length(precision)
+ .precision(precision, scale)
+ .nullable(!qryEntity.getNotNullFields().contains(fieldName));
+
+ String dirtyColumnName = qryEntity.getAliases().getOrDefault(fieldName, fieldName);
+ String columnName = sanitizeColumnName(dirtyColumnName);
+ colDefinitions[defIdx++] = ColumnDefinition.column(columnName, colType);
+
+ if (qryEntity.getKeyFields().contains(fieldName)) {
+ pkColumnNames[pkIdx++] = columnName;
+ }
+
+ fieldNameForColumnMappings.put(columnName, fieldName);
+ }
+
+ var table = TableDefinition.builder(tableName)
+ .schema(schema)
+ .columns(colDefinitions)
+ .primaryKey(pkColumnNames)
+ .build();
+
+ // TODO: Test one of these are null;
+ @Nullable Map.Entry<String, String> typeHints = Map.entry(qryEntity.getKeyType(), qryEntity.getValueType());
+ return new GenerateTableResult(table, fieldNameForColumnMappings, typeHints);
+ }
+
+ public TableDefinition generateTableDefinition(CacheConfiguration<?, ?> cacheCfg) throws FieldNameConflictException {
+ return generate(cacheCfg).tableDefinition;
+ }
+
+ private QueryEntity getOrCreateQueryEntity(CacheConfiguration cacheCfg) throws FieldNameConflictException {
+ // TODO: Map the whole object and key instead of the query entities
+ QueryEntity qe;
+
+ Map.Entry<Class<?>, Class<?>> typeHints = null;
+ try {
+ typeHints = this.tableTypeRegistry.typesForTable(cacheCfg.getName());
+ } catch (ClassNotFoundException ex) {
+ LOGGER.error("Found TableTypeHint for cache but one of the class was not in the Classpath: {}", cacheCfg.getName(), ex);
+ }
+
+ if (typeHints != null) {
+ LOGGER.warn("Found TableTypeHint for cache: {}:{}", cacheCfg.getName(), typeHints);
+ qe = new QueryEntity(typeHints.getKey(), typeHints.getValue());
+ } else if (cacheCfg.getQueryEntities().isEmpty()) {
+ // This should return a KeyValue Binary Cache
+ // TODO: Check if the value type should be null or not.
+ qe = new QueryEntity();
+ var binaryClsName = byte[].class.getName();
+ qe.setKeyType(binaryClsName);
+ qe.setValueType(binaryClsName);
+
+ // TODO: Check this default precision for binary caches
+ Map<String, Integer> precision = new HashMap<>();
+ precision.put("ID", DEFAULT_BINARY_FIELD_LENGTH);
+ precision.put("VAL", DEFAULT_BINARY_FIELD_LENGTH);
+ qe.setFieldsPrecision(precision);
+ } else if (cacheCfg.getQueryEntities().size() == 1) {
+ qe = new QueryEntity((QueryEntity) cacheCfg.getQueryEntities().iterator().next());
+ } else {
+ LOGGER.warn("Unexpected number of entities (Only 0, 1 QueryEntity is support ATM): {}:{}", cacheCfg.getName(),
+ cacheCfg.getQueryEntities().size());
+ // TODO: Throw a better checked exception
+ throw new RuntimeException("Unsupported number of queryEntities in cache configuration: " + cacheCfg.getQueryEntities().size());
+ }
+
+ return populateQueryEntity(qe, allowExtraFields);
+ }
+
+ enum FieldType {
+ PRIMITIVE,
+ ARRAY,
+ POJO_ATTRIBUTE
+ }
+
+ static class InspectedField {
+ @Nullable
+ private String fieldName;
+
+ private String typeName;
+
+ private FieldType fieldType;
+
+ private boolean nullable;
+
+ public InspectedField(@Nullable String fieldName, String typeName, FieldType fieldType) {
+ this(fieldName, typeName, fieldType, !(fieldType == FieldType.PRIMITIVE || fieldType == FieldType.ARRAY));
+ }
+
+ public InspectedField(@Nullable String fieldName, String typeName, FieldType fieldType, boolean nullable) {
+ this.fieldName = fieldName;
+ this.typeName = typeName;
+ this.fieldType = fieldType;
+ this.nullable = nullable;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public FieldType getFieldType() {
+ return fieldType;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ @Override
+ public String toString() {
+ return "InspectedField{"
+ + "fieldName='" + fieldName + '\''
+ + ", typeName='" + typeName + '\''
+ + ", fieldType=" + fieldType
+ + ", nullable=" + nullable
+ + '}';
+ }
+ }
+
+ /** GenerateTableResult. */
+ public static class GenerateTableResult {
+ private final TableDefinition tableDefinition;
+
+ private final Map<String, String> fieldNameForColumn;
+
+ @Nullable
+ private final Map.Entry<String, String> rawTypeHints;
+
+ /**
+ * Constructor.
+ *
+ * @param tableDefinition Table definition.
+ * @param fieldNameForColumn Mapping of columns to their corresponding field names.
+ * @param rawTypeHints Mapping of type hints by column.
+ */
+ public GenerateTableResult(TableDefinition tableDefinition, Map<String, String> fieldNameForColumn,
+ Map.Entry<String, String> rawTypeHints) {
+ this.tableDefinition = tableDefinition;
+ this.fieldNameForColumn = fieldNameForColumn;
+ this.rawTypeHints = rawTypeHints;
+ }
+
+ public TableDefinition tableDefinition() {
+ return tableDefinition;
+ }
+
+ public Map<String, String> fieldNameForColumnMappings() {
+ return fieldNameForColumn;
+ }
+
+ @Nullable
+ public Map.Entry<String, String> typeHints() {
+ return rawTypeHints;
+ }
+ }
+
+ private static class FieldNameCandidateSupplier implements Supplier<String> {
+
+ private final String[] base;
+
+ private final Function<Integer, String> additional;
+
+ private int idx;
+
+ public FieldNameCandidateSupplier(String[] base, Function<Integer, String> additional) {
+ this.base = base;
+ this.additional = additional;
+ this.idx = 0;
+ }
+
+ @Override
+ public String get() {
+ String ret = (idx < base.length) ? base[idx] : additional.apply(idx - base.length);
+ idx++;
+ return ret;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/Namespace.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/Namespace.java
new file mode 100644
index 00000000000..86344372275
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/Namespace.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tablemanagement;
+
+/** Old class to help navigate the limitation of custom schemas in Ignite 3. */
+public class Namespace {
+ private static String MIGRATION_TOOLS_SCHEMA = "MIGRATION_TOOLS_PVT";
+
+ /**
+ * Resolves the full table name for a table inside our namespace.
+ *
+ * @param tableName Unquoted table name.
+ * @return The full unquoted full table name (with the namespace prefix).
+ */
+ public static String resolveTableName(String tableName) {
+ return MIGRATION_TOOLS_SCHEMA + "_" + tableName;
+ }
+
+ /**
+ * Check if the provided table name is inside our namespace.
+ *
+ * @param tableName Name of the table to check. May be quoted or unquoted.
+ * @return Whether the table is inside our namespace or not.
+ */
+ public static boolean isTableFromNamespace(String tableName) {
+ if (tableName.charAt(0) == '"') {
+ tableName = tableName.substring(1);
+ }
+
+ return tableName.startsWith(MIGRATION_TOOLS_SCHEMA + "_");
+ }
+
+ private Namespace() {
+ // Intentionally left blank.
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/PersistentTableTypeRegistryImpl.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/PersistentTableTypeRegistryImpl.java
new file mode 100644
index 00000000000..430f92aa876
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/PersistentTableTypeRegistryImpl.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tablemanagement;
+
+import static org.apache.ignite3.catalog.definitions.ColumnDefinition.column;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite3.catalog.ColumnType;
+import org.apache.ignite3.catalog.definitions.TableDefinition;
+import org.apache.ignite3.client.IgniteClient;
+import org.apache.ignite3.table.KeyValueView;
+import org.apache.ignite3.table.Table;
+import org.jetbrains.annotations.Nullable;
+
+/** {@link TableTypeRegistry} implementation backed by a table. */
+public class PersistentTableTypeRegistryImpl implements TableTypeRegistry {
+ private static final String TABLE_NAME = Namespace.resolveTableName("TABLETYPEREGISTRY");
+
+ private final IgniteClient client;
+
+ private final CompletableFuture<KeyValueView<String, TableTypeRecord>> tableFuture;
+
+ public PersistentTableTypeRegistryImpl(IgniteClient client) {
+ this.client = client;
+ this.tableFuture = initTable().thenApply(table -> table.keyValueView(String.class, TableTypeRecord.class));
+ }
+
+ private CompletableFuture<Table> initTable() {
+ // TODO: GG-39800 Add logs
+ // TODO: GG-39801 Add a private schema if AI3 when supports it.
+ TableDefinition def = TableDefinition.builder(TABLE_NAME)
+ .ifNotExists()
+ .columns(
+ column("TABLE_KEY", ColumnType.VARCHAR),
+ column("keyClass", ColumnType.VARCHAR),
+ column("valClass", ColumnType.VARCHAR))
+ .primaryKey("TABLE_KEY")
+ .build();
+
+ return this.client.catalog().createTableAsync(def);
+ }
+
+ @Override
+ public @Nullable Map.Entry<Class<?>, Class<?>> typesForTable(String tableName) throws ClassNotFoundException {
+ var table = this.tableFuture.join();
+ var types = table.get(null, tableName);
+
+ if (types == null) {
+ return null;
+ } else {
+ var keyClass = Class.forName(types.keyClass);
+ var valClass = Class.forName(types.valClass);
+ return Map.entry(keyClass, valClass);
+ }
+ }
+
+ @Override
+ public void registerTypesForTable(String tableName, Map.Entry<String, String> tableTypes) {
+ var table = this.tableFuture.join();
+ var record = new TableTypeRecord(tableTypes.getKey(), tableTypes.getValue());
+ table.put(null, tableName, record);
+ }
+
+ private static class TableTypeRecord {
+ private String keyClass;
+
+ private String valClass;
+
+ public TableTypeRecord() {
+ // Intentionally left blank
+ }
+
+ public TableTypeRecord(String keyClass, String valClass) {
+ this.keyClass = keyClass;
+ this.valClass = valClass;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/RegisterOnlyTableTypeRegistry.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/RegisterOnlyTableTypeRegistry.java
new file mode 100644
index 00000000000..063a55f406f
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/RegisterOnlyTableTypeRegistry.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tablemanagement;
+
+import java.util.Map;
+import org.jetbrains.annotations.Nullable;
+
+/** Decorator for {@link TableTypeRegistry} that only registers new hints. */
+public class RegisterOnlyTableTypeRegistry implements TableTypeRegistry {
+ private final TableTypeRegistry base;
+
+ public RegisterOnlyTableTypeRegistry(TableTypeRegistry base) {
+ this.base = base;
+ }
+
+ @Override
+ @Nullable
+ public Map.Entry<Class<?>, Class<?>> typesForTable(String tableName) throws ClassNotFoundException {
+ return null;
+ }
+
+ @Override
+ public void registerTypesForTable(String tableName, Map.Entry<String, String> tableTypes) {
+ base.registerTypesForTable(tableName, tableTypes);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/SchemaUtils.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/SchemaUtils.java
new file mode 100644
index 00000000000..f6ad556107d
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/SchemaUtils.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tablemanagement;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite3.internal.client.table.ClientSchema;
+import org.apache.ignite3.internal.client.table.ClientTable;
+
+/** Utility methods to interact with internal client schemas. */
+public class SchemaUtils {
+ private static final Method GET_LATEST_SCHEMA_METHOD;
+
+ static {
+ try {
+ GET_LATEST_SCHEMA_METHOD = ClientTable.class.getDeclaredMethod("getLatestSchema");
+ GET_LATEST_SCHEMA_METHOD.setAccessible(true);
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private SchemaUtils() {
+ // Intentionally left blank
+ }
+
+ /** Exposes {@link ClientTable#getLatestSchema()}. */
+ public static CompletableFuture<ClientSchema> getLatestSchemaForTable(ClientTable clientTable) {
+ try {
+ return (CompletableFuture<ClientSchema>) GET_LATEST_SCHEMA_METHOD.invoke(clientTable);
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/TableTypeRegistry.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/TableTypeRegistry.java
new file mode 100644
index 00000000000..b9fb83484d5
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/TableTypeRegistry.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tablemanagement;
+
+import java.util.Map;
+import org.jetbrains.annotations.Nullable;
+
+/** This interface provides a registry for mappings between tables and their corresponding java types. */
+public interface TableTypeRegistry {
+ /**
+ * Gets the available type hints for the table.
+ *
+ * @param tableName Must be escaped according to Ignite 3 rules.
+ * @return The type hints for the table, or null if none are available.
+ */
+ @Nullable Map.Entry<Class<?>, Class<?>> typesForTable(String tableName) throws ClassNotFoundException;
+
+ /**
+ * Registers the supplied type hints for the given table. Existing hints will be replaced.
+ *
+ * @param tableName Must be escaped according to Ignite 3 rules.
+ * @param tableTypes Entry with the ClassNames of Key and Value types, as returned by {@link Class#getName()}.
+ */
+ void registerTypesForTable(String tableName, Map.Entry<String, String> tableTypes);
+
+ static Map.Entry<String, String> typeHints(Class<?> keyType, Class<?> valType) {
+ return Map.entry(keyType.getName(), valType.getName());
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/TableTypeRegistryMapImpl.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/TableTypeRegistryMapImpl.java
new file mode 100644
index 00000000000..109e3f1cf8f
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/tablemanagement/TableTypeRegistryMapImpl.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.tablemanagement;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.jetbrains.annotations.Nullable;
+
+/** {@link TableTypeRegistry} implementation based on a in-memory map. */
+public class TableTypeRegistryMapImpl implements TableTypeRegistry {
+ private final Map<String, Map.Entry<String, String>> hints = new HashMap<>();
+
+ @Override
+ public @Nullable Map.Entry<Class<?>, Class<?>> typesForTable(String tableName) {
+ var typeNames = hints.get(tableName);
+ if (typeNames == null) {
+ return null;
+ }
+
+ try {
+ return Map.entry(Class.forName(typeNames.getKey()), Class.forName(typeNames.getValue()));
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void registerTypesForTable(String tableName, Map.Entry<String, String> tableTypes) {
+ this.hints.putIfAbsent(tableName, tableTypes);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/CalendarToInstantConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/CalendarToInstantConverter.java
new file mode 100644
index 00000000000..b5de3511d37
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/CalendarToInstantConverter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts Java Calendars to Instants. */
+public class CalendarToInstantConverter implements TypeConverter<Calendar, Instant> {
+ @Override
+ public Instant toColumnType(Calendar calendar) throws Exception {
+ return calendar.toInstant();
+ }
+
+ @Override
+ public Calendar toObjectType(Instant instant) throws Exception {
+ return GregorianCalendar.from(ZonedDateTime.ofInstant(instant, ZoneId.systemDefault()));
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/CalendarToLocalDateTimeConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/CalendarToLocalDateTimeConverter.java
new file mode 100644
index 00000000000..225b25f5af8
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/CalendarToLocalDateTimeConverter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts Java Calendars to LocalDateTime. */
+public class CalendarToLocalDateTimeConverter implements TypeConverter<Calendar, LocalDateTime> {
+ @Override
+ public LocalDateTime toColumnType(Calendar calendar) throws Exception {
+ return LocalDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC);
+ }
+
+ @Override
+ public Calendar toObjectType(LocalDateTime dateTime) throws Exception {
+ return GregorianCalendar.from(ZonedDateTime.of(dateTime, ZoneOffset.UTC));
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/DateToInstantConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/DateToInstantConverter.java
new file mode 100644
index 00000000000..608d4d2cea6
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/DateToInstantConverter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.time.Instant;
+import java.util.Date;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts Java Date Calendars to Instants. */
+public class DateToInstantConverter implements TypeConverter<Date, Instant> {
+ @Override
+ public Instant toColumnType(Date instant) throws Exception {
+ return instant.toInstant();
+ }
+
+ @Override
+ public Date toObjectType(Instant instant) throws Exception {
+ return Date.from(instant);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/DateToLocalDateTimeConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/DateToLocalDateTimeConverter.java
new file mode 100644
index 00000000000..d3886901d39
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/DateToLocalDateTimeConverter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Date;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts Java Date to LocalDateTime. */
+public class DateToLocalDateTimeConverter implements TypeConverter<Date, LocalDateTime> {
+ @Override
+ public LocalDateTime toColumnType(Date date) throws Exception {
+ return LocalDateTime.ofInstant(date.toInstant(), ZoneOffset.UTC);
+ }
+
+ @Override
+ public Date toObjectType(LocalDateTime dateTime) throws Exception {
+ return Date.from(dateTime.toInstant(ZoneOffset.UTC));
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/EnumToStringTypeConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/EnumToStringTypeConverter.java
new file mode 100644
index 00000000000..72fb84a9721
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/EnumToStringTypeConverter.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts an Enum to String. */
+public class EnumToStringTypeConverter<T extends Enum<T>> implements TypeConverter<Enum<T>, String> {
+ private final Class<T> enumType;
+
+ public EnumToStringTypeConverter(Class<T> type) {
+ enumType = type;
+ }
+
+ @Override
+ public String toColumnType(Enum<T> anEnum) throws Exception {
+ return anEnum.name();
+ }
+
+ @Override
+ public Enum<T> toObjectType(String name) throws Exception {
+ return Enum.valueOf(enumType, name);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/IdentityConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/IdentityConverter.java
new file mode 100644
index 00000000000..5a047331d00
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/IdentityConverter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Identity Converter. */
+public class IdentityConverter<T> implements TypeConverter<T, T> {
+ public static IdentityConverter INSTANCE = new IdentityConverter();
+
+ @Override
+ public T toColumnType(T t) throws Exception {
+ return t;
+ }
+
+ @Override
+ public T toObjectType(T t) throws Exception {
+ return t;
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlDateToLocalDateConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlDateToLocalDateConverter.java
new file mode 100644
index 00000000000..798188108d8
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlDateToLocalDateConverter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.sql.Date;
+import java.time.LocalDate;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts SQL Date to LocalDateTime. */
+public class SqlDateToLocalDateConverter implements TypeConverter<Date, LocalDate> {
+ @Override
+ public LocalDate toColumnType(Date date) throws Exception {
+ return date.toLocalDate();
+ }
+
+ @Override
+ public Date toObjectType(LocalDate date) throws Exception {
+ return Date.valueOf(date);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimeToLocalTimeConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimeToLocalTimeConverter.java
new file mode 100644
index 00000000000..01d33ae0146
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimeToLocalTimeConverter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.sql.Time;
+import java.time.LocalTime;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts SQL Time to Local Time. */
+public class SqlTimeToLocalTimeConverter implements TypeConverter<Time, LocalTime> {
+ @Override
+ public LocalTime toColumnType(Time time) throws Exception {
+ return time.toLocalTime();
+ }
+
+ @Override
+ public Time toObjectType(LocalTime localTime) throws Exception {
+ return Time.valueOf(localTime);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimestampToInstantConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimestampToInstantConverter.java
new file mode 100644
index 00000000000..9d783ea33f2
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimestampToInstantConverter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.sql.Timestamp;
+import java.time.Instant;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts SQL Timestamp to Instant. */
+public class SqlTimestampToInstantConverter implements TypeConverter<Timestamp, Instant> {
+ @Override
+ public Instant toColumnType(Timestamp timestamp) throws Exception {
+ return timestamp.toInstant();
+ }
+
+ @Override
+ public Timestamp toObjectType(Instant time) throws Exception {
+ return Timestamp.from(time);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimestampToLocalDateTimeConverter.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimestampToLocalDateTimeConverter.java
new file mode 100644
index 00000000000..7fe10cb0420
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/SqlTimestampToLocalDateTimeConverter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import org.apache.ignite3.table.mapper.TypeConverter;
+
+/** Converts SQL Timestamp to LocalDateTime. */
+public class SqlTimestampToLocalDateTimeConverter implements TypeConverter<Timestamp, LocalDateTime> {
+ @Override
+ public LocalDateTime toColumnType(Timestamp timestamp) throws Exception {
+ return timestamp.toLocalDateTime();
+ }
+
+ @Override
+ public Timestamp toObjectType(LocalDateTime time) throws Exception {
+ return Timestamp.valueOf(time);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/StaticTypeConverterFactory.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/StaticTypeConverterFactory.java
new file mode 100644
index 00000000000..21ea8084c67
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/StaticTypeConverterFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite3.table.mapper.TypeConverter;
+import org.jetbrains.annotations.Nullable;
+
+/** Static Factory for {@link TypeConverter}. Supports some predefined types and Enums. */
+public class StaticTypeConverterFactory implements TypeConverterFactory {
+ public static final TypeConverterFactory DEFAULT_INSTANCE = new StaticTypeConverterFactory();
+
+ private static final Map<Map.Entry<Class<?>, Class<?>>, TypeConverter<?, ?>> CONVERTERS = new HashMap<>(
+ Map.of(
+ Map.entry(Calendar.class, Instant.class), new CalendarToInstantConverter(),
+ Map.entry(Calendar.class, LocalDateTime.class), new CalendarToLocalDateTimeConverter(),
+ Map.entry(Date.class, Instant.class), new DateToInstantConverter(),
+ Map.entry(Date.class, LocalDateTime.class), new DateToLocalDateTimeConverter(),
+ Map.entry(java.sql.Date.class, LocalDate.class), new SqlDateToLocalDateConverter(),
+ Map.entry(Time.class, LocalTime.class), new SqlTimeToLocalTimeConverter(),
+ Map.entry(Timestamp.class, Instant.class), new SqlTimestampToInstantConverter(),
+ Map.entry(Timestamp.class, LocalDateTime.class), new SqlTimestampToLocalDateTimeConverter()
+ )
+ );
+
+ private StaticTypeConverterFactory() {
+ // Intentionally left blank.
+ }
+
+ @Override
+ public @Nullable <ObjectT, ColumnT> TypeConverter<? extends ObjectT, ? extends ColumnT> converterFor(Class<ObjectT> objType,
+ Class<ColumnT> columnType) {
+ return (TypeConverter<? extends ObjectT, ? extends ColumnT>)
+ CONVERTERS.computeIfAbsent(Map.entry(objType, columnType), StaticTypeConverterFactory::tryCreateNewConverter);
+ }
+
+ @Nullable
+ private static TypeConverter<?, ?> tryCreateNewConverter(Map.Entry<Class<?>, Class<?>> entry) {
+ Class<?> objType = entry.getKey();
+ if (objType.isEnum()) {
+ return new EnumToStringTypeConverter<>((Class<? extends Enum>) objType);
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/TypeConverterFactory.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/TypeConverterFactory.java
new file mode 100644
index 00000000000..b4ff3fb6605
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/types/converters/TypeConverterFactory.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.types.converters;
+
+import org.apache.ignite3.table.mapper.TypeConverter;
+import org.jetbrains.annotations.Nullable;
+
+/** Factory for {@link org.apache.ignite3.table.mapper.TypeConverter} based on the required types. */
+public interface TypeConverterFactory {
+ @Nullable
+ <ObjectT, ColumnT> TypeConverter<? extends ObjectT, ? extends ColumnT> converterFor(Class<ObjectT> objType, Class<ColumnT> columnType);
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/utils/ClassnameUtils.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/utils/ClassnameUtils.java
new file mode 100644
index 00000000000..7bec229eb49
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite/migrationtools/utils/ClassnameUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.utils;
+
+import java.util.Map;
+
+/** Small utilities for handling class names. */
+public final class ClassnameUtils {
+
+ public static final Map<String, String> PRIMITIVE_TO_WRAPPER = Map.of(
+ boolean.class.getName(), Boolean.class.getName(),
+ byte.class.getName(), Byte.class.getName(),
+ char.class.getName(), Character.class.getName(),
+ short.class.getName(), Short.class.getName(),
+ int.class.getName(), Integer.class.getName(),
+ long.class.getName(), Long.class.getName(),
+ double.class.getName(), Double.class.getName(),
+ float.class.getName(), Float.class.getName(),
+ void.class.getName(), Void.class.getName()
+ );
+
+ private ClassnameUtils() {
+ // Intentionally left blank
+ }
+
+ public static String ensureWrapper(String className) {
+ return PRIMITIVE_TO_WRAPPER.getOrDefault(className, className);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite3/internal/catalog/sql/CatalogExtensions.java b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite3/internal/catalog/sql/CatalogExtensions.java
new file mode 100644
index 00000000000..a49d9bbdaed
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/main/java/org/apache/ignite3/internal/catalog/sql/CatalogExtensions.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite3.internal.catalog.sql;
+
+import org.apache.ignite3.catalog.definitions.TableDefinition;
+
+/** Utility methods based on the Ignite 3 Catalog package. */
+public class CatalogExtensions {
+ private CatalogExtensions() {
+ // Intentionally left blank.
+ }
+
+ public static String sqlFromTableDefinition(TableDefinition def) {
+ return new CreateFromDefinitionImpl(null).from(def).toString();
+ }
+}
diff --git a/migration-tools/modules/migration-tools-commons/src/test/java/org/apache/ignite/migrationtools/sql/sql/SqlDdlGeneratorTest.java b/migration-tools/modules/migration-tools-commons/src/test/java/org/apache/ignite/migrationtools/sql/sql/SqlDdlGeneratorTest.java
new file mode 100644
index 00000000000..f8f45f45c35
--- /dev/null
+++ b/migration-tools/modules/migration-tools-commons/src/test/java/org/apache/ignite/migrationtools/sql/sql/SqlDdlGeneratorTest.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.sql.sql;
+
+import static java.util.Map.entry;
+import static org.apache.ignite.migrationtools.sql.SqlDdlGenerator.EXTRA_FIELDS_COLUMN_NAME;
+import static org.apache.ignite.migrationtools.sql.sql.SqlDdlGeneratorTest.ColumnRecord.nonKey;
+import static org.apache.ignite.migrationtools.sql.sql.SqlDdlGeneratorTest.ColumnRecord.primaryKey;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.groups.Tuple.tuple;
+import static org.junit.jupiter.api.Named.named;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.time.DayOfWeek;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.BiFunction;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.commons.collections4.IterableUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.examples.model.Organization;
+import org.apache.ignite.examples.model.Person;
+import org.apache.ignite.migrationtools.sql.SqlDdlGenerator;
+import org.apache.ignite.migrationtools.tablemanagement.TableTypeRegistryMapImpl;
+import org.apache.ignite.migrationtools.tests.models.ComplexKeyIntStr;
+import org.apache.ignite.migrationtools.tests.models.SimplePojo;
+import org.apache.ignite3.catalog.ColumnSorted;
+import org.apache.ignite3.catalog.definitions.ColumnDefinition;
+import org.apache.ignite3.catalog.definitions.TableDefinition;
+import org.assertj.core.api.SoftAssertions;
+import org.junit.jupiter.api.Named;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.FieldSource;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class SqlDdlGeneratorTest {
+ // TODO: Need to create tests to check if the module is populating the typehints correctly.
+
+ private static final List<Named<Boolean>> EXTRA_FIELDS_ENABLED_ARG = List.of(
+ named("No extra fields support", false),
+ named("With extra fields support", true)
+ );
+
+ private static final List<ColumnRecord> PERSON_EXPECTED_FIELDS = List.of(
+ primaryKey("KEY", "INT"),
+ nonKey("id", "BIGINT", true),
+ nonKey("orgId", "BIGINT", true),
+ nonKey("firstName", "VARCHAR", true),
+ nonKey("lastName", "VARCHAR", true),
+ nonKey("resume", "VARCHAR", true),
+ nonKey("salary", "DOUBLE", false)
+ );
+
+ private static final QueryEntity POJO_WITH_PRIMITIVES_QE = new QueryEntity(Object.class.getName(), Object.class.getName())
+ .setFields(
+ Stream.of(
+ entry("ID", long.class.getName()),
+ entry("NAME", String.class.getName()),
+ entry("AGE", int.class.getName()),
+ entry("SALARY", double.class.getName())
+ ).collect(linkedMapCollector())
+ )
+ .setKeyFields(new HashSet<>(Collections.singleton("ID")));
+
+ private static final List<ColumnRecord> POJO_WITH_PRIMITIVES_FIELDS = List.of(
+ primaryKey("ID", "BIGINT"),
+ nonKey("NAME", "VARCHAR", true),
+ nonKey("AGE", "INT", true),
+ nonKey("SALARY", "DOUBLE", true)
+ );
+
+ static CacheConfiguration configWithIndexType(Class<?> keyType, Class<?> valType) {
+ CacheConfiguration cacheCfg = new CacheConfiguration();
+ cacheCfg.setIndexedTypes(keyType, valType);
+ return cacheCfg;
+ }
+
+ static CacheConfiguration configWithQeKeyValue(Class<?> keyType, Class<?> valType) {
+ CacheConfiguration cacheCfg = new CacheConfiguration();
+ QueryEntity qe = new QueryEntity(keyType, valType);
+ cacheCfg.setQueryEntities(Collections.singleton(qe));
+ return cacheCfg;
+ }
+
+ static TableDefinition generateTableDef(CacheConfiguration cacheCfg, boolean allowExtraFields) {
+ SqlDdlGenerator gen = new SqlDdlGenerator(new TableTypeRegistryMapImpl(), allowExtraFields);
+ return gen.generateTableDefinition(cacheCfg);
+ }
+
+ static List<Arguments> provideSupportedClasses() {
+ // TODO: Check lengths
+ List<Map.Entry<Class, String>> primitives = List.of(
+ entry(boolean.class, "BOOLEAN"),
+ entry(byte.class, "TINYINT"),
+ entry(char.class, "CHAR"),
+ entry(short.class, "SMALLINT"),
+ entry(int.class, "INT"),
+ entry(long.class, "BIGINT"),
+ entry(double.class, "DOUBLE"),
+ entry(float.class, "REAL"),
+ entry(byte[].class, "VARBINARY"),
+ // More complex stuff
+ entry(UUID.class, "UUID"),
+ entry(BitSet.class, "VARBINARY"),
+ entry(LocalTime.class, "TIME"),
+ entry(LocalDate.class, "DATE"),
+ entry(LocalDateTime.class, "TIMESTAMP"),
+ entry(Instant.class, "TIMESTAMP WITH LOCAL TIME ZONE"),
+ entry(Date.class, "DATE"),
+ entry(Time.class, "TIME"),
+ entry(Timestamp.class, "TIMESTAMP"),
+ entry(DayOfWeek.class, "VARCHAR")
+ );
+
+ // These types are only supported on the value side.
+ List<Map.Entry<Class, String>> supportedOnlyHasValues = List.of(
+ entry(boolean[].class, "VARBINARY"),
+ entry(char[].class, "VARBINARY"),
+ entry(short[].class, "VARBINARY"),
+ entry(int[].class, "VARBINARY"),
+ entry(long[].class, "VARBINARY"),
+ entry(double[].class, "VARBINARY"),
+ entry(float[].class, "VARBINARY"),
+ entry(List.class, "VARBINARY")
+ );
+
+ List<Arguments> ret = new ArrayList<>(primitives.size() * primitives.size());
+ for (var cacheCfgArgs : provideCacheConfigSupplier()) {
+ for (var keyTypeRef : primitives) {
+ for (var valTypeRef : IterableUtils.chainedIterable(primitives, supportedOnlyHasValues)) {
+ var args = arguments(
+ Stream.concat(
+ Arrays.stream(cacheCfgArgs.get()),
+ Stream.of(keyTypeRef.getKey(), keyTypeRef.getValue(), valTypeRef.getKey(), valTypeRef.getValue())
+ ).toArray());
+
+ ret.add(args);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ static List<Arguments> provideCacheConfigSupplier() {
+ var cfgGenerators = List.of(
+ named("From indexType", (BiFunction<Class<?>, Class<?>, CacheConfiguration>) SqlDdlGeneratorTest::configWithIndexType),
+ named("From QE key and value fields",
+ (BiFunction<Class<?>, Class<?>, CacheConfiguration>) SqlDdlGeneratorTest::configWithQeKeyValue)
+ );
+
+ List<Arguments> ret = new ArrayList<>(EXTRA_FIELDS_ENABLED_ARG.size() * cfgGenerators.size());
+ for (var extraFieldProp : EXTRA_FIELDS_ENABLED_ARG) {
+ for (var cfgGen : cfgGenerators) {
+ ret.add(arguments(cfgGen, extraFieldProp));
+ }
+ }
+
+ return ret;
+ }
+
+ private static <K> UnaryOperator<K> skipNth(int numElements, UnaryOperator<K> operator) {
+ return new UnaryOperator<>() {
+ int counter = 0;
+
+ @Override
+ public K apply(K k) {
+ return (counter++ < numElements) ? k : operator.apply(k);
+ }
+ };
+ }
+
+ private static void testCacheConfig(CacheConfiguration<?, ?> cacheCfg, boolean allowExtraFields, List<ColumnRecord> asserts) {
+ var tableDef = generateTableDef(cacheCfg, allowExtraFields);
+ Stream<ColumnRecord> allowFieldsCol = (allowExtraFields)
+ ? Stream.of(new ColumnRecord(EXTRA_FIELDS_COLUMN_NAME, "VARBINARY", true, false))
+ : Stream.empty();
+
+ var expectedColumns = Stream.concat(asserts.stream(), allowFieldsCol)
+ .map(e -> tuple(e.name, e.type, e.nullable))
+ .collect(Collectors.toList());
+
+ var expectedPrimaryKeys = asserts.stream()
+ .filter(e -> e.isPk)
+ .map(e -> e.name)
+ .collect(Collectors.toList());
+
+ SoftAssertions sa = new SoftAssertions();
+
+ sa.assertThat(tableDef.primaryKeyColumns())
+ .as("Primary Keys")
+ .extracting(ColumnSorted::columnName)
+ .containsExactlyElementsOf(expectedPrimaryKeys);
+
+ sa.assertThat(tableDef.columns())
+ .as("Columns: (name, type, nullable)")
+ .extracting(ColumnDefinition::name, d -> d.type().typeName(), d -> d.type().nullable())
+ .containsExactlyInAnyOrderElementsOf(expectedColumns);
+
+ sa.assertAll();
+ }
+
+ private static <K, V> Collector<Map.Entry<K, V>, ?, LinkedHashMap<K, V>> linkedMapCollector() {
+ return Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
+ (a, b) -> {
+ throw new RuntimeException("Should never happen");
+ },
+ LinkedHashMap::new);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideSupportedClasses")
+ void testTableDefUsingIndexedTypes(
+ BiFunction<Class<?>, Class<?>, CacheConfiguration> cacheConfigSupplier,
+ boolean allowExtraFields,
+ Class keyType,
+ String keyDef,
+ Class valType,
+ String valDef
+ ) {
+ var cacheCfg = configWithIndexType(keyType, valType);
+ testCacheConfig(cacheCfg, false, List.of(
+ primaryKey("ID", keyDef),
+ nonKey("VAL", valDef, false)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideCacheConfigSupplier")
+ void testTableDefWithComplexKeyAndSimplePojo(BiFunction<Class<?>, Class<?>, CacheConfiguration> cacheConfigSupplier,
+ boolean allowExtraFields) {
+ var cacheCfg = cacheConfigSupplier.apply(ComplexKeyIntStr.class, SimplePojo.class);
+ {
+ QueryEntity qe = (QueryEntity) cacheCfg.getQueryEntities().stream().findFirst().orElseThrow();
+ Map<String, String> aliases = qe.getAliases();
+ // We currently cannot make an alias to switch the field case due to an hack to support case-insensitive mappings.
+ // aliases.put("id", "ID");
+ aliases.put("affinityStr", "AFFINITY_STR");
+ }
+
+ testCacheConfig(cacheCfg, allowExtraFields, List.of(
+ primaryKey("id", "INT"),
+ primaryKey("AFFINITY_STR", "VARCHAR"),
+ nonKey("name", "VARCHAR", true),
+ nonKey("amount", "INT", false),
+ nonKey("decimalAmount", "DECIMAL", true)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideCacheConfigSupplier")
+ void testTableDefWithOrganizationPojo(BiFunction<Class<?>, Class<?>, CacheConfiguration> cacheConfigSupplier,
+ boolean allowExtraFields) {
+ var cacheCfg = cacheConfigSupplier.apply(int.class, Organization.class);
+ testCacheConfig(cacheCfg, allowExtraFields, List.of(
+ primaryKey("KEY", "INT"),
+ nonKey("id", "BIGINT", true),
+ nonKey("name", "VARCHAR", true),
+ nonKey("type", "VARCHAR", true),
+ nonKey("lastUpdated", "TIMESTAMP", true)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideCacheConfigSupplier")
+ void testTableDefWithPersonPojo(BiFunction<Class<?>, Class<?>, CacheConfiguration> cacheConfigSupplier, boolean allowExtraFields) {
+ // TODO: Make dynamic pojos with BB to cover all the possible scenarios...
+ var cacheCfg = cacheConfigSupplier.apply(int.class, Person.class);
+ testCacheConfig(cacheCfg, allowExtraFields, PERSON_EXPECTED_FIELDS);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideCacheConfigSupplier")
+ void testTableDefWithPersonPojoNotInClasspath(BiFunction<Class<?>, Class<?>, CacheConfiguration> cacheConfigSupplier,
+ boolean allowExtraFields) {
+ var cacheCfg = cacheConfigSupplier.apply(int.class, Person.class);
+ QueryEntity qe = (QueryEntity) cacheCfg.getQueryEntities().stream().findFirst().orElseThrow();
+ qe.setValueType(Person.class.getName().replace("Person", "FakePerson"));
+
+ Set<String> notNullFields = new HashSet<>();
+ notNullFields.add("salary");
+ qe.setNotNullFields(notNullFields);
+
+ testCacheConfig(cacheCfg, allowExtraFields, PERSON_EXPECTED_FIELDS);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideCacheConfigSupplier")
+ void testCasingMismatchBetweenQueryEntityAndClass(BiFunction<Class<?>, Class<?>, CacheConfiguration> cacheConfigSupplier,
+ boolean allowExtraFields) {
+ var cacheCfg = cacheConfigSupplier.apply(int.class, Person.class);
+ QueryEntity qe = (QueryEntity) cacheCfg.getQueryEntities().stream().findFirst().orElseThrow();
+ // Skips the key, randomize the casing for the field name.
+ qe.setFields(
+ qe.getFields().entrySet().stream()
+ .map(e -> entry(StringUtils.swapCase(e.getKey()), e.getValue()))
+ .collect(linkedMapCollector())
+ );
+
+ var renameExpectedColumns = PERSON_EXPECTED_FIELDS.stream()
+ .map(skipNth(1, c -> new ColumnRecord(StringUtils.swapCase(c.name), c.type, c.nullable, c.isPk)))
+ .collect(Collectors.toList());
+
+ testCacheConfig(cacheCfg, allowExtraFields, renameExpectedColumns);
+ }
+
+ @ParameterizedTest
+ @FieldSource("EXTRA_FIELDS_ENABLED_ARG")
+ void testTableDefWithPojoWithPrimitiveFieldsDefined(boolean allowExtraFields) {
+ CacheConfiguration<?, ?> cacheCfg = new CacheConfiguration<>("some-cache");
+ cacheCfg.setQueryEntities(Collections.singletonList(POJO_WITH_PRIMITIVES_QE));
+
+ testCacheConfig(cacheCfg, allowExtraFields, POJO_WITH_PRIMITIVES_FIELDS);
+ }
+
+ @ParameterizedTest
+ @FieldSource("EXTRA_FIELDS_ENABLED_ARG")
+ void testTableDefWithPojoWithPrimitiveFieldsDefinedInTypes(boolean allowExtraFields) {
+ CacheConfiguration<?, ?> cacheCfg = new CacheConfiguration<>("some-cache");
+ cacheCfg.setQueryEntities(Collections.singletonList(
+ new QueryEntity(POJO_WITH_PRIMITIVES_QE).setKeyType(long.class.getName()).setValueType("PersonRecordClassInRoot")));
+
+ testCacheConfig(cacheCfg, allowExtraFields, POJO_WITH_PRIMITIVES_FIELDS);
+ }
+
+ @Test
+ void checkPrintUsingStub() {
+ var cacheCfg = configWithIndexType(int.class, Person.class);
+ var tableDef = generateTableDef(cacheCfg, false);
+
+ var sqlStr = SqlDdlGenerator.createDdlQuery(tableDef);
+ assertThat(sqlStr).isNotBlank();
+ }
+
+ static class ColumnRecord {
+ String name;
+
+ String type;
+
+ boolean nullable;
+
+ boolean isPk;
+
+ public ColumnRecord(String name, String type, boolean nullable, boolean isPk) {
+ this.name = name;
+ this.type = type;
+ this.nullable = nullable;
+ this.isPk = isPk;
+ }
+
+ static ColumnRecord primaryKey(String name, String type) {
+ return new ColumnRecord(name, type, false, true);
+ }
+
+ static ColumnRecord nonKey(String name, String type, boolean nullable) {
+ return new ColumnRecord(name, type, nullable, false);
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/build.gradle b/migration-tools/modules/migration-tools-config-converter/build.gradle
new file mode 100644
index 00000000000..3b19dafe4b7
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/build.gradle
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+apply from: "$rootDir/buildscripts/java-integration-test.gradle"
+apply from: "$rootDir/buildscripts/java-test-fixtures.gradle"
+
+description = 'migration-tools-config-converter'
+
+dependencies {
+ implementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ implementation project(":migration-tools-commons")
+ implementation libs.ignite2.core
+ implementation libs.ignite2.spring
+ implementation libs.commons.lang3
+ implementation libs.commons.collections4
+ implementation libs.mockito.core
+ implementation libs.slf4j.api
+
+ testImplementation testFixtures(project(":migration-tools-config-converter"))
+ testImplementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ testImplementation project(":migration-tools-commons-tests")
+ testImplementation libs.assertj.core
+ testImplementation libs.junit5.api
+ testImplementation libs.junit5.params
+ testImplementation libs.typesafe.config
+
+ integrationTestImplementation testFixtures(project(":migration-tools-config-converter"))
+ integrationTestImplementation project(":migration-tools-commons-tests")
+ integrationTestImplementation libs.assertj.core
+ integrationTestImplementation libs.junit5.api
+ integrationTestImplementation libs.junit5.params
+ integrationTestImplementation libs.testcontainers.core
+ integrationTestImplementation libs.testcontainers.junit
+
+ testFixturesImplementation libs.commons.lang3
+ testFixturesImplementation libs.typesafe.config
+}
+
+tasks.integrationTest.dependsOn += ":packaging:docker"
diff --git a/migration-tools/modules/migration-tools-config-converter/src/integrationTest/java/org/apache/ignite/migrationtools/config/ClusterLoadsConfigTest.java b/migration-tools/modules/migration-tools-config-converter/src/integrationTest/java/org/apache/ignite/migrationtools/config/ClusterLoadsConfigTest.java
new file mode 100644
index 00000000000..1e3f601cf03
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/integrationTest/java/org/apache/ignite/migrationtools/config/ClusterLoadsConfigTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.stream.Stream;
+import org.apache.ignite.migrationtools.tests.containers.Ignite3ClusterContainer;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.MountableFile;
+
+/** Tests loading the converted configurations in a cluster. */
+public class ClusterLoadsConfigTest {
+ private static Stream<String> allCacheExamples() {
+ return ConfigExamples.configPaths();
+ }
+
+ @ParameterizedTest
+ @MethodSource("allCacheExamples")
+ void dockerRunsSuccessfully(String inputPath) throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile(inputPath);
+ var nodeCfgPath = testDescr.getNodeCfgPath();
+
+ try (var container = new GenericContainer(Ignite3ClusterContainer.DOCKER_IMAGE_NAME)
+ .withCommand("--config-path=/node.cfg")
+ .withCopyToContainer(MountableFile.forHostPath(nodeCfgPath), "/node.cfg")
+ .withStartupCheckStrategy(new IsRunningStartupCheckStrategy())
+ .waitingFor(Wait.forLogMessage(".*Components started.*", 1)
+ .withStartupTimeout(Duration.of(30, ChronoUnit.SECONDS)))) {
+
+ container.start();
+ assertThat(container.isRunning()).isTrue();
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/ConfigurationConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/ConfigurationConverter.java
new file mode 100644
index 00000000000..2327cf1a6bc
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/ConfigurationConverter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import static org.mockito.internal.util.MockUtil.typeMockabilityOf;
+
+import java.beans.PropertyDescriptor;
+import java.io.File;
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.commons.collections4.MultiValuedMap;
+import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.converters.ClientConnectorConverter;
+import org.apache.ignite.migrationtools.config.converters.DataRegionConfigurationConverter;
+import org.apache.ignite.migrationtools.config.converters.DiscoverySpiConverter;
+import org.apache.ignite.migrationtools.config.converters.SslContextFactoryConverter;
+import org.apache.ignite.migrationtools.config.loader.CustomBeanFactory;
+import org.jetbrains.annotations.Nullable;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.core.io.FileSystemResource;
+
+/** ConfigurationConverter. */
+public class ConfigurationConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationConverter.class);
+
+ // TODO: Make this injected.
+ private static List<org.apache.ignite.migrationtools.config.converters.ConfigurationConverter> converters = List.of(
+ // new CommunicationSpiConverter(), Disabled the TcpCommSpi. Will use the port from the Discovery Module
+ new DiscoverySpiConverter(),
+ new SslContextFactoryConverter(),
+ new ClientConnectorConverter(),
+ new DataRegionConfigurationConverter()
+ );
+
+ /**
+ * Converts an Apache Ignite 2 configuration file into its Apache Ignite 3 node and cluster configurations counterparts.
+ *
+ * @param inputFile Apache Ignite 2 configuration file.
+ * @param nodeCfgFile Target Ignite 3 node configuration file.
+ * @param clusterCfgFile Target Ignite 3 cluster configuration file.
+ * @param includeDefaults Whether defaults will be included in the target files.
+ * @param clientClassloader Custom classloader to load any additional 3rd party libraries (Optional).
+ * @throws Exception Well this is embarrassing.
+ */
+ public static void convertConfigurationFile(File inputFile, File nodeCfgFile, File clusterCfgFile,
+ boolean includeDefaults, @Nullable ClassLoader clientClassloader) throws Exception {
+ MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> propertyTracker = new HashSetValuedHashMap<>();
+ CustomBeanFactory beanFactory = new CustomBeanFactory(propertyTracker);
+ IgniteConfiguration igniteCfg =
+ Ignite2ConfigurationUtils.loadIgnite2Configuration(new FileSystemResource(inputFile), beanFactory, clientClassloader);
+ igniteCfg = spyIgniteConfiguration(propertyTracker, igniteCfg, IgniteConfiguration.class);
+
+ var registry = Ignite3ConfigurationUtils.loadCombinedRegistry(nodeCfgFile.toPath(), clusterCfgFile.toPath(), includeDefaults);
+ registry.onDefaultsPersisted().get(10, TimeUnit.SECONDS);
+ LOGGER.info("Finished loading configurations into memory");
+
+ LOGGER.info("Starting to map configurations");
+ for (org.apache.ignite.migrationtools.config.converters.ConfigurationConverter converter : converters) {
+ converter.convert(igniteCfg, registry);
+ }
+
+ // List the remaining properties in the tracker.
+ for (Map.Entry<Object, Collection<Pair<PropertyDescriptor, BeanDefinition>>> e : propertyTracker.asMap().entrySet()) {
+ List<String> propertyNames = e.getValue().stream().map(p -> p.getKey().getName()).collect(Collectors.toList());
+ LOGGER.warn("Could not convert properties: {} - {}", e.getKey().getClass().getName(), propertyNames);
+ }
+
+ LOGGER.info("Finished mapping configurations. Waiting until files are written...");
+ registry.stopAsync().join();
+ }
+
+ private static <T> T spyIgniteConfiguration(
+ MultiValuedMap<Object, Pair<PropertyDescriptor, BeanDefinition>> propertyTracker, T obj, Class<T> klass) {
+ return Mockito.mock(klass, Mockito.withSettings()
+ .spiedInstance(obj)
+ .defaultAnswer(ans -> {
+ // Check if the method was registered
+ Method calledMethod = ans.getMethod();
+ propertyTracker.get(obj)
+ .stream()
+ .filter(e -> Objects.equals(e.getLeft().getReadMethod(), calledMethod))
+ .findFirst()
+ .ifPresent(e -> {
+ LOGGER.info("Registered property access: {} - {}::{}", obj.hashCode(), e.getLeft().getReadMethod(),
+ calledMethod);
+ // We will remove the property from the tracker.
+ propertyTracker.removeMapping(obj, e);
+ });
+
+ // Spy object recursively
+ Object ret = ans.callRealMethod();
+ if (ret != null && typeMockabilityOf(ret.getClass(), null).mockable()) {
+ return spyIgniteConfiguration(propertyTracker, (T) ret, (Class<T>) ret.getClass());
+ } else {
+ return ret;
+ }
+ }));
+ }
+
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/ClientConnectorConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/ClientConnectorConverter.java
new file mode 100644
index 00000000000..a5a6aac19b5
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/ClientConnectorConverter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.converters;
+
+import java.util.concurrent.ExecutionException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.registry.ConfigurationRegistryInterface;
+import org.apache.ignite3.client.handler.configuration.ClientConnectorExtensionConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** ClientConnectorConverter. */
+public class ClientConnectorConverter implements ConfigurationConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ClientConnectorConverter.class);
+
+ @Override
+ public void convert(IgniteConfiguration src,
+ ConfigurationRegistryInterface registry) throws ExecutionException, InterruptedException {
+ var clientConnCfg = src.getClientConnectorConfiguration();
+ if (clientConnCfg == null) {
+ LOGGER.warn("Could not find a ClientConnectorConfiguration in the source configuration.");
+ return;
+ }
+
+
+ var target = registry.getConfiguration(ClientConnectorExtensionConfiguration.KEY).clientConnector();
+
+ target.port().update(clientConnCfg.getPort()).get();
+
+ if (clientConnCfg.getHandshakeTimeout() != org.apache.ignite.configuration.ClientConnectorConfiguration.DFLT_HANDSHAKE_TIMEOUT) {
+ target.connectTimeoutMillis().update((int) Math.min(Integer.MAX_VALUE, clientConnCfg.getHandshakeTimeout())).get();
+ }
+
+ if (clientConnCfg.getIdleTimeout() > 0) {
+ target.idleTimeoutMillis().update(clientConnCfg.getIdleTimeout()).get();
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/CommunicationSpiConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/CommunicationSpiConverter.java
new file mode 100644
index 00000000000..4df0819221c
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/CommunicationSpiConverter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.converters;
+
+import java.util.concurrent.ExecutionException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.registry.ConfigurationRegistryInterface;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite3.internal.network.configuration.NetworkExtensionConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** CommunicationSpiConverter. */
+public class CommunicationSpiConverter implements ConfigurationConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(CommunicationSpiConverter.class);
+
+ @Override public void convert(IgniteConfiguration src,
+ ConfigurationRegistryInterface registry) throws ExecutionException, InterruptedException {
+ var commSpi = src.getCommunicationSpi();
+ if (commSpi == null) {
+ LOGGER.error("Could not find a CommunicationSpi in the source configuration.");
+ return;
+ }
+
+ if (!(commSpi instanceof TcpCommunicationSpi)) {
+ LOGGER.warn("CommunicationSpi is not a TcpCommunicationSpi: {}", commSpi.getClass().getName());
+ return;
+ }
+
+ TcpCommunicationSpi tcpCommSpi = (TcpCommunicationSpi) commSpi;
+ var target = registry.getConfiguration(NetworkExtensionConfiguration.KEY).network();
+
+ target.port().update(tcpCommSpi.getLocalPort()).get();
+ if (tcpCommSpi.getLocalPortRange() > 0) {
+ // TODO: Check if we can implement an additional policy for migrating the port range.
+ LOGGER.error("Local Port Range in TcpCommunicationSpi will be ignored. There's no similar feature in Apache Ignite 3");
+ }
+
+ // TODO: Check if this is the correct delay config
+ target.inbound().tcpNoDelay().update(tcpCommSpi.isTcpNoDelay()).get();
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/ConfigurationConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/ConfigurationConverter.java
new file mode 100644
index 00000000000..843be09d1e7
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/ConfigurationConverter.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.converters;
+
+import java.util.concurrent.ExecutionException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.registry.ConfigurationRegistryInterface;
+
+/** ConfigurationConverter. */
+public interface ConfigurationConverter {
+ void convert(IgniteConfiguration src,
+ ConfigurationRegistryInterface registry) throws ExecutionException, InterruptedException;
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/DataRegionConfigurationConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/DataRegionConfigurationConverter.java
new file mode 100644
index 00000000000..a1446ce7b3e
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/DataRegionConfigurationConverter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.converters;
+
+import java.util.concurrent.ExecutionException;
+import java.util.function.Consumer;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.registry.ConfigurationRegistryInterface;
+import org.apache.ignite3.internal.pagememory.configuration.schema.PersistentPageMemoryProfileChange;
+import org.apache.ignite3.internal.pagememory.configuration.schema.VolatilePageMemoryProfileChange;
+import org.apache.ignite3.internal.storage.configurations.StorageConfiguration;
+import org.apache.ignite3.internal.storage.configurations.StorageExtensionConfiguration;
+import org.apache.ignite3.internal.storage.configurations.StorageProfileChange;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** DataRegionConfigurationConverter. */
+public class DataRegionConfigurationConverter implements ConfigurationConverter {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(DataRegionConfigurationConverter.class);
+
+ @SuppressWarnings("checkstyle:linelength")
+ private static void convertDataRegion(
+ ConfigurationRegistryInterface registry,
+ DataRegionConfiguration regionCfg, boolean isDefault) throws ExecutionException, InterruptedException {
+ String regionName = (!isDefault) ? regionCfg.getName() : DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
+ if (!regionCfg.isPersistenceEnabled()) {
+ convertMemoryRegion(registry, regionCfg, regionName);
+ } else {
+ convertPersistentRegion(registry, regionCfg, regionName);
+ }
+ }
+
+ private static void convertPersistentRegion(ConfigurationRegistryInterface registry,
+ DataRegionConfiguration regionCfg,
+ String regionName) throws ExecutionException, InterruptedException {
+ StorageConfiguration storageConfig = registry.getConfiguration(StorageExtensionConfiguration.KEY).storage();
+
+ Consumer<StorageProfileChange> changer = t -> t.convert(PersistentPageMemoryProfileChange.class)
+ .changeSizeBytes(regionCfg.getMaxSize())
+ .changeReplacementMode(regionCfg.getPageReplacementMode().name());
+
+ storageConfig.profiles().change(c -> c.createOrUpdate(regionName, changer)).get();
+ // TODO: Configure the allocator
+ }
+
+ private static void convertMemoryRegion(ConfigurationRegistryInterface registry,
+ DataRegionConfiguration regionCfg,
+ String regionName) throws ExecutionException, InterruptedException {
+ StorageConfiguration storageConfig = registry.getConfiguration(StorageExtensionConfiguration.KEY).storage();
+
+ // TODO: Check where the eviction configuration went!!
+ // TODO: Check where the changeEmptyPagesPoolSize configuration went!!
+ Consumer<StorageProfileChange> changer = t -> t.convert(VolatilePageMemoryProfileChange.class)
+ .changeInitSizeBytes(regionCfg.getInitialSize())
+ .changeMaxSizeBytes(regionCfg.getMaxSize());
+
+ storageConfig.profiles().change(c -> c.createOrUpdate(regionName, changer)).get();
+ // TODO: Configure the allocator
+ }
+
+ // TODO: Clean-up the code.
+ @Override
+ public void convert(IgniteConfiguration src,
+ ConfigurationRegistryInterface registry) throws ExecutionException, InterruptedException {
+ DataStorageConfiguration dataStorageCfg = src.getDataStorageConfiguration();
+ if (dataStorageCfg == null) {
+ LOGGER.warn("Could not find data storage configuration");
+ return;
+ }
+
+ // TODO: I'm not sure how to change the pageSize afterwards.
+ if (dataStorageCfg.getPageSize() != 0) {
+ LOGGER.warn("Unable to force pageSize to :{}. Unsupported in Ignite 3", dataStorageCfg.getPageSize());
+ }
+
+ DataRegionConfiguration defaultRegionCfg = dataStorageCfg.getDefaultDataRegionConfiguration();
+ convertDataRegion(registry, defaultRegionCfg, true);
+
+ DataRegionConfiguration[] otherRegionCfgs = dataStorageCfg.getDataRegionConfigurations();
+ if (otherRegionCfgs != null) {
+ for (DataRegionConfiguration regionCfg : otherRegionCfgs) {
+ convertDataRegion(registry, regionCfg, false);
+ }
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/DiscoverySpiConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/DiscoverySpiConverter.java
new file mode 100644
index 00000000000..78e9e6d9354
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/DiscoverySpiConverter.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.converters;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.registry.ConfigurationRegistryInterface;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite3.internal.network.configuration.NetworkExtensionConfiguration;
+import org.apache.ignite3.internal.network.configuration.StaticNodeFinderConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** DiscoverySpiConverter. */
+public class DiscoverySpiConverter implements ConfigurationConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DiscoverySpiConverter.class);
+
+ private static Pattern PORT_RANGE_PATTERN = Pattern.compile(":(?<fp>\\d+)(..(?<lp>\\d+))?$");
+
+ private static Collection<String> collectAddresses(String rawAddress) {
+ // This method is very optimistic because the validation is performed by the configuration spi.
+ String finalRawAddr = rawAddress.trim();
+ Matcher m = PORT_RANGE_PATTERN.matcher(finalRawAddr);
+ if (m.find()) {
+ int fp = Integer.parseInt(m.group("fp"));
+ String lpStr = m.group("lp");
+ if (lpStr == null) {
+ // Just a single port, add as is.
+ return Collections.singleton(finalRawAddr);
+ } else {
+ // Port Range
+ String hostname = finalRawAddr.substring(0, m.start());
+ int lp = Integer.parseInt(lpStr);
+ return IntStream.rangeClosed(fp, lp).mapToObj(p -> hostname + ":" + p).collect(Collectors.toList());
+ }
+ } else {
+ // It's just an hostname, we add the ai2 default port.
+ return Collections.singleton(finalRawAddr + ":" + 47500);
+ }
+ }
+
+ @Override
+ public void convert(IgniteConfiguration src,
+ ConfigurationRegistryInterface registry) throws ExecutionException, InterruptedException {
+ var discSpi = src.getDiscoverySpi();
+ if (discSpi == null) {
+ LOGGER.error("Could not find a CommunicationSpi in the source configuration.");
+ return;
+ }
+
+ if (!(discSpi instanceof TcpDiscoverySpi)) {
+ LOGGER.warn("CommunicationSpi is not a TcpDiscoverySpi: {}", discSpi.getClass().getName());
+ return;
+ }
+
+ TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discSpi;
+ var target = registry.getConfiguration(NetworkExtensionConfiguration.KEY).network();
+
+ try {
+ // tcpDiscoverySpi.getLocalPort() offers an unexpected behaviour. This should be caught by the tests.
+ int locPort = (int) FieldUtils.readDeclaredField(tcpDiscoverySpi, "locPort", true);
+ target.port().update(locPort).get();
+ } catch (IllegalAccessException e) {
+ LOGGER.error("Error getting TcpDiscoverySpi local port", e);
+ }
+
+ if (tcpDiscoverySpi.getLocalPortRange() > 0) {
+ // TODO: Check if we can implement an additional policy for migrating the port range.
+ LOGGER.error("Local Port Range in TcpDiscoverySpi will be ignored. There's no similar feature in Apache Ignite 3");
+ }
+
+ if (tcpDiscoverySpi.getIpFinder() instanceof TcpDiscoveryVmIpFinder) {
+ try {
+ // We cannot use the #getRegisteredAddresses because it will filter out addresses that cannot be solve during runtime.
+ var rawAddresses = (Collection<String>) FieldUtils.readField(tcpDiscoverySpi.getIpFinder(), "addrs", true);
+ var addrArr = rawAddresses.stream()
+ .flatMap(a -> collectAddresses(a).stream())
+ .toArray(String[]::new);
+
+ ((StaticNodeFinderConfiguration) target.nodeFinder())
+ .netClusterNodes()
+ .update(addrArr)
+ .get();
+ } catch (IllegalAccessException | ClassCastException e) {
+ LOGGER.error("Error getting addresses from TcpDiscoveryVmIpFinder addrs", e);
+ }
+ } else {
+ LOGGER.warn("Only TcpDiscoveryVmIpFinder configurations are supported, found: {}",
+ Optional.ofNullable(tcpDiscoverySpi.getIpFinder()).map(c -> c.getClass().getName()).orElse("None"));
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/SslContextFactoryConverter.java b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/SslContextFactoryConverter.java
new file mode 100644
index 00000000000..42d6c623d94
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/main/java/org/apache/ignite/migrationtools/config/converters/SslContextFactoryConverter.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config.converters;
+
+import java.util.concurrent.ExecutionException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.registry.ConfigurationRegistryInterface;
+import org.apache.ignite.ssl.SslContextFactory;
+import org.apache.ignite3.client.handler.configuration.ClientConnectorExtensionConfiguration;
+import org.apache.ignite3.internal.network.configuration.NetworkExtensionConfiguration;
+import org.apache.ignite3.internal.network.configuration.SslConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** SslContextFactoryConverter. */
+public class SslContextFactoryConverter implements ConfigurationConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SslContextFactoryConverter.class);
+
+ @SuppressWarnings("PMD.UseEqualsToCompareStrings")
+ private static void applySslConfigs(SslConfiguration target, SslContextFactory tcpCommSpi)
+ throws InterruptedException, ExecutionException {
+ target.keyStore().path().update(tcpCommSpi.getKeyStoreFilePath()).get();
+ target.keyStore().password().update(new String(tcpCommSpi.getKeyStorePassword())).get();
+ if (tcpCommSpi.getTrustStoreFilePath() != null) {
+ target.trustStore().path().update(tcpCommSpi.getTrustStoreFilePath()).get();
+ }
+ if (tcpCommSpi.getTrustStorePassword() != null) {
+ target.trustStore().password().update(new String(tcpCommSpi.getTrustStorePassword())).get();
+ }
+
+ // Checking for reference on purpose to guess if the value was defaulted or not.
+ if (tcpCommSpi.getKeyAlgorithm() != SslContextFactory.DFLT_KEY_ALGORITHM) {
+ target.ciphers().update(tcpCommSpi.getKeyAlgorithm());
+ }
+
+ // Check if ssl validation was disabled.
+ var thrustManagers = tcpCommSpi.getTrustManagers();
+ if (thrustManagers != null && thrustManagers.length == 1
+ && thrustManagers[0].getClass().equals(SslContextFactory.getDisabledTrustManager().getClass())) {
+ target.clientAuth().update("NONE").get();
+ }
+ }
+
+ @Override
+ public void convert(IgniteConfiguration src,
+ ConfigurationRegistryInterface registry) throws ExecutionException, InterruptedException {
+ var srcSslCf = src.getSslContextFactory();
+ if (srcSslCf == null) {
+ LOGGER.error("Could not find a CommunicationSpi in the source configuration.");
+ return;
+ }
+
+ if (!(srcSslCf instanceof SslContextFactory)) {
+ LOGGER.warn("SslContextFactory is not a SslContextFactory: {}", srcSslCf.getClass().getName());
+ return;
+ }
+
+ SslContextFactory sslCtxFactory = (SslContextFactory) srcSslCf;
+
+ var discoTarget = registry.getConfiguration(NetworkExtensionConfiguration.KEY).network().ssl();
+ applySslConfigs(discoTarget, sslCtxFactory);
+
+ var jdbcTarget = registry.getConfiguration(ClientConnectorExtensionConfiguration.KEY).clientConnector().ssl();
+ applySslConfigs(jdbcTarget, sslCtxFactory);
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/test/java/org/apache/ignite/migrationtools/config/ConfigurationConverterTest.java b/migration-tools/modules/migration-tools-config-converter/src/test/java/org/apache/ignite/migrationtools/config/ConfigurationConverterTest.java
new file mode 100644
index 00000000000..963aa640db8
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/test/java/org/apache/ignite/migrationtools/config/ConfigurationConverterTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.typesafe.config.Config;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class ConfigurationConverterTest {
+ private static Stream<String> configPaths() {
+ return ConfigExamples.configPaths();
+ }
+
+ private static Stream<Arguments> nodePorts() {
+ return Stream.of(
+ Arguments.of("configs-custom/ignite-config.0.xml", 49500),
+ Arguments.of("configs-custom/ignite-config.1.xml", 49501),
+ Arguments.of("configs-custom/ignite-config.2.xml", 40500)
+ );
+ }
+
+ private static Stream<Arguments> apiPorts() {
+ return Stream.of(
+ Arguments.of("configs-custom/ignite-config.0.xml", null),
+ Arguments.of("configs-custom/ignite-config.1.xml", null),
+ Arguments.of("configs-custom/ignite-config.2.xml", 10801)
+ );
+ }
+
+ private static Stream<Arguments> dataStorageConfigs() {
+ return Stream.of(
+ Arguments.of("configs-custom/ignite-config.0.xml", List.of(
+ (Consumer<Map.Entry<String, Map<String, Object>>>) e -> {
+ var name = e.getKey();
+ var conf = e.getValue();
+
+ assertThat(name).isEqualTo("default");
+ assertThat(conf.get("engine")).isEqualTo("aimem");
+ assertThat(conf.get("initSizeBytes")).isEqualTo(100 * 1024 * 1024);
+ },
+ (Consumer<Map.Entry<String, Map<String, Object>>>) e -> {
+ var name = e.getKey();
+ var conf = e.getValue();
+
+ assertThat(name).isEqualTo("40MB_Region_Eviction");
+ assertThat(conf.get("engine")).isEqualTo("aimem");
+ assertThat(conf.get("initSizeBytes")).isEqualTo(20 * 1024 * 1024);
+ assertThat(conf.get("maxSizeBytes")).isEqualTo(40 * 1024 * 1024);
+ }
+ )),
+ Arguments.of("configs-custom/ignite-config.1.xml", List.of(
+ (Consumer<Map.Entry<String, Map<String, Object>>>) e -> {
+ var name = e.getKey();
+ var conf = e.getValue();
+
+ assertThat(name).isEqualTo("default");
+ assertThat(conf.get("engine")).isEqualTo("aimem");
+ assertThat(conf.get("initSizeBytes")).isEqualTo(50 * 1024 * 1024);
+ assertThat(conf.get("maxSizeBytes")).isEqualTo(150 * 1024 * 1024);
+ }
+ )),
+ Arguments.of("configs-custom/ignite-config.2.xml", List.of(
+ (Consumer<Map.Entry<String, Map<String, Object>>>) e -> {
+ var name = e.getKey();
+ var conf = e.getValue();
+
+ assertThat(name).isEqualTo("default");
+ assertThat(conf.get("engine")).isEqualTo("aipersist");
+ assertThat(conf.get("sizeBytes")).isEqualTo(50 * 1024 * 1024);
+ }
+ ))
+ );
+ }
+
+ private static Stream<Arguments> clusterNodes() {
+ var cfg0 = Stream.concat(IntStream.rangeClosed(49500, 49520).boxed(), IntStream.rangeClosed(4500, 5000).boxed())
+ .map(i -> "127.0.0.1:" + i)
+ .collect(Collectors.toList());
+
+ return Stream.of(
+ Arguments.of("configs-custom/ignite-config.0.xml", cfg0),
+ Arguments.of("configs-custom/ignite-config.1.xml", List.of("node1:49500", "node2:49500", "node3:49500")),
+ Arguments.of("configs-custom/ignite-config.2.xml", List.of("node1.my-company.com:47500", "node2.my-company.com:47500"))
+ );
+ }
+
+ private static Stream<Arguments> sslConfigs() {
+ Consumer<Config> notConfigured = c -> assertThat(c.hasPath("network.ssl")).isFalse();
+
+ BiConsumer<Config, String> c1 = (c, module) -> {
+ assertThat(c.getString(module + ".ssl.keyStore.password")).isEqualTo("123456");
+ assertThat(c.getString(module + ".ssl.keyStore.path")).isEqualTo("keystore/node.jks");
+ assertThat(c.getString(module + ".ssl.trustStore.password")).isEqualTo("123456");
+ assertThat(c.getString(module + ".ssl.trustStore.path")).isEqualTo("keystore/trust.jks");
+ assertThat(c.hasPath(module + ".ssl.clientAuth")).isFalse();
+ assertThat(c.hasPath(module + ".ssl.ciphers")).isFalse();
+ };
+
+ BiConsumer<Config, String> c2 = (c, module) -> {
+ assertThat(c.getString(module + ".ssl.ciphers")).isEqualTo("Blowfish");
+ assertThat(c.getString(module + ".ssl.clientAuth")).isEqualTo("NONE");
+ assertThat(c.getString(module + ".ssl.keyStore.password")).isEqualTo("123456");
+ assertThat(c.getString(module + ".ssl.keyStore.path")).isEqualTo("keystore/node.jks");
+ };
+
+ Function<BiConsumer<Config, String>, Consumer<Config>> binder = checker -> cfg -> {
+ checker.accept(cfg, "network");
+ checker.accept(cfg, "clientConnector");
+ };
+
+ return Stream.of(
+ Arguments.of("configs-custom/ignite-config.0.xml", binder.apply(c1)),
+ Arguments.of("configs-custom/ignite-config.1.xml", notConfigured),
+ Arguments.of("configs-custom/ignite-config.2.xml", binder.apply(c2))
+ );
+ }
+
+ @Order(1)
+ @ParameterizedTest
+ @MethodSource("configPaths")
+ void noExceptionIsThrownWhenConverting(String inputPath) throws Exception {
+ ConfigTestUtils.loadResourceFile(inputPath);
+ }
+
+ @ParameterizedTest
+ @MethodSource("nodePorts")
+ void testNodePort(String inputPath, Integer expectedNodePort) throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile(inputPath);
+ Config cfg = testDescr.getNodeCfg().getConfig("ignite");
+ if (expectedNodePort != null) {
+ var port = cfg.getInt("network.port");
+ assertThat(port).isEqualTo(expectedNodePort);
+ } else {
+ assertThat(cfg.hasPath("network.port")).isFalse();
+ }
+ }
+
+ @ParameterizedTest(name = "[{index}] {0}")
+ @MethodSource("clusterNodes")
+ void testClusterNodes(String inputPath, List<String> expectedClusterNodes) throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile(inputPath);
+ Config cfg = testDescr.getNodeCfg().getConfig("ignite");
+ var key = "network.nodeFinder.netClusterNodes";
+ if (expectedClusterNodes != null) {
+ var port = cfg.getStringList(key);
+ assertThat(port).containsExactlyElementsOf(expectedClusterNodes);
+ } else {
+ assertThat(cfg.hasPath(key)).isFalse();
+ }
+ }
+
+ @ParameterizedTest(name = "[{index}] {0}")
+ @MethodSource("sslConfigs")
+ void testSslConfigs(String inputPath, Consumer<Config> configAssertions) throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile(inputPath);
+ Config cfg = testDescr.getNodeCfg().getConfig("ignite");
+ configAssertions.accept(cfg);
+ }
+
+ @ParameterizedTest
+ @MethodSource("apiPorts")
+ void testClientApiPort(String inputPath, Integer expectedPort) throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile(inputPath);
+ Config cfg = testDescr.getNodeCfg().getConfig("ignite");
+ if (expectedPort != null) {
+ var port = cfg.getInt("clientConnector.port");
+ assertThat(port).isEqualTo(expectedPort);
+ } else {
+ assertThat(cfg.hasPath("clientConnector.port")).isFalse();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("dataStorageConfigs")
+ void testDataStorageConfigurations(String inputPath, List<Consumer<Map.Entry<String, Map<String, Object>>>> reqs) throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile(inputPath);
+ Config cfg = testDescr.getNodeCfg().getConfig("ignite");
+
+ var profiles = cfg.getObject("storage.profiles").unwrapped().entrySet();
+ assertThat(profiles).satisfiesExactlyInAnyOrder(reqs.toArray(new Consumer[0]));
+ }
+
+ @Test
+ void testDefaultsAreNotWritten() throws Exception {
+ var testDescr = ConfigTestUtils.loadResourceFile("configs-custom/empty-config.xml");
+ assertThat(Files.size(testDescr.getNodeCfgPath())).isZero();
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/testFixtures/java/org/apache/ignite/migrationtools/config/ConfigExamples.java b/migration-tools/modules/migration-tools-config-converter/src/testFixtures/java/org/apache/ignite/migrationtools/config/ConfigExamples.java
new file mode 100644
index 00000000000..e18f6f3b1f3
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/testFixtures/java/org/apache/ignite/migrationtools/config/ConfigExamples.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import java.util.stream.Stream;
+
+/** Defines the location of all the configuration examples used in the tests. */
+public class ConfigExamples {
+ private ConfigExamples() {
+ // Intentionally left blank.
+ }
+
+ /**
+ * Returns all the know paths of the example configurations used in the tests.
+ *
+ * @return The configurations paths.
+ */
+ public static Stream<String> configPaths() {
+ return Stream.of(
+ "configs-custom/ignite-config.0.xml",
+ "configs-custom/ignite-config.1.xml",
+ "configs-custom/ignite-config.2.xml"
+ );
+ }
+}
diff --git a/migration-tools/modules/migration-tools-config-converter/src/testFixtures/java/org/apache/ignite/migrationtools/config/ConfigTestUtils.java b/migration-tools/modules/migration-tools-config-converter/src/testFixtures/java/org/apache/ignite/migrationtools/config/ConfigTestUtils.java
new file mode 100644
index 00000000000..b3a045149da
--- /dev/null
+++ b/migration-tools/modules/migration-tools-config-converter/src/testFixtures/java/org/apache/ignite/migrationtools/config/ConfigTestUtils.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.config;
+
+import static org.apache.ignite.migrationtools.config.ConfigurationConverter.convertConfigurationFile;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.commons.lang3.concurrent.Memoizer;
+import org.apache.commons.lang3.function.Failable;
+
+/** ConfigTestUtils. */
+public class ConfigTestUtils {
+ private static final Path resourcesPath = Path.of("../../resources");
+
+ private static Memoizer<String, ConfigTestDescriptor> cachedDescriptors =
+ new Memoizer<>(Failable.asFunction(ConfigTestUtils::doLoadResourceFile), false);
+
+ private ConfigTestUtils() {
+ // Intentionally left blank
+ }
+
+ private static ConfigTestDescriptor doLoadResourceFile(String inputPath) throws Exception {
+ Path inputCfgPath = resourcesPath.resolve(inputPath);
+ Path nodeCfgPath = Files.createTempFile("node", ".cfg");
+ Path clusterCfgPath = Files.createTempFile("cluster", ".cfg");
+
+ convertConfigurationFile(inputCfgPath.toFile(), nodeCfgPath.toFile(), clusterCfgPath.toFile(), false, null);
+
+ String nodeCfgStr = Files.readString(nodeCfgPath, StandardCharsets.UTF_8);
+ String clusterCfgStr = Files.readString(clusterCfgPath, StandardCharsets.UTF_8);
+
+ Config nodeCfg = ConfigFactory.parseString(nodeCfgStr);
+ Config clusterCfg = ConfigFactory.parseString(clusterCfgStr);
+
+ return new ConfigTestDescriptor(
+ inputCfgPath,
+ nodeCfgPath,
+ clusterCfgPath,
+ nodeCfg,
+ clusterCfg
+ );
+ }
+
+ public static ConfigTestDescriptor loadResourceFile(String inputPath) throws Exception {
+ return cachedDescriptors.compute(inputPath);
+ }
+
+ /** ConfigTestDescriptor. */
+ public static class ConfigTestDescriptor {
+ // AI Config File
+ private final Path sourceFile;
+
+ private final Path nodeCfgPath;
+
+ private final Path clusterCfgPath;
+
+ private final Config nodeCfg;
+
+ private final Config clusterCfg;
+
+ /** Constructor. */
+ public ConfigTestDescriptor(Path sourceFile, Path nodeCfgPath, Path clusterCfgPath, Config nodeCfg, Config clusterCfg) {
+ this.sourceFile = sourceFile;
+ this.nodeCfgPath = nodeCfgPath;
+ this.clusterCfgPath = clusterCfgPath;
+ this.nodeCfg = nodeCfg;
+ this.clusterCfg = clusterCfg;
+ }
+
+ public Path getSourceFile() {
+ return sourceFile;
+ }
+
+ public Path getNodeCfgPath() {
+ return nodeCfgPath;
+ }
+
+ public Path getClusterCfgPath() {
+ return clusterCfgPath;
+ }
+
+ public Config getNodeCfg() {
+ return nodeCfg;
+ }
+
+ public Config getClusterCfg() {
+ return clusterCfg;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-persistence/build.gradle b/migration-tools/modules/migration-tools-persistence/build.gradle
new file mode 100644
index 00000000000..565596f3de2
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/build.gradle
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply from: "$rootDir/buildscripts/java-core.gradle"
+apply from: "$rootDir/buildscripts/publishing.gradle"
+apply from: "$rootDir/buildscripts/java-junit5.gradle"
+apply from: "$rootDir/buildscripts/java-integration-test.gradle"
+
+description = 'migration-tools-persistence'
+
+dependencies {
+ implementation project(":migration-tools-commons")
+ implementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ implementation libs.ignite2.core
+ implementation libs.ignite2.spring
+ implementation libs.commons.lang3
+ implementation libs.commons.collections4
+ implementation libs.jackson.databind
+ implementation libs.slf4j.api
+ compileOnly libs.spotbugs.annotations
+
+ testImplementation project(":migration-tools-commons-tests")
+ testImplementation project(":migration-tools-e2e-core")
+ testImplementation project(":migration-tools-e2e-implementations")
+ testImplementation testFixtures(project(":migration-tools-ai2-test-fixtures"))
+ testImplementation libs.mockito.core
+ testImplementation libs.assertj.core
+ testImplementation libs.junit5.api
+ testImplementation libs.junit5.params
+
+ integrationTestImplementation project(":migration-tools-commons")
+ integrationTestImplementation project(path: ":migration-tools-ignite3-repack", configuration: "shadow")
+ integrationTestImplementation project(":migration-tools-commons-tests")
+ integrationTestImplementation project(":migration-tools-e2e-core")
+ integrationTestImplementation project(":migration-tools-e2e-implementations")
+ integrationTestImplementation testFixtures(project(":migration-tools-ai2-test-fixtures"))
+ integrationTestImplementation libs.ignite2.core
+ integrationTestImplementation libs.commons.lang3
+ integrationTestImplementation libs.commons.collections4
+ integrationTestImplementation libs.jackson.databind
+ integrationTestImplementation libs.mockito.core
+ integrationTestImplementation libs.assertj.core
+ integrationTestImplementation libs.junit5.api
+ integrationTestImplementation libs.junit5.params
+ integrationTestImplementation libs.testcontainers.core
+ integrationTestImplementation libs.testcontainers.junit
+}
+
+tasks.integrationTest.dependsOn += ":migration-tools-e2e-ai2-runner:docker"
diff --git a/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/BasePersistentTestContext.java b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/BasePersistentTestContext.java
new file mode 100644
index 00000000000..eb891588e4d
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/BasePersistentTestContext.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.junit.platform.commons.support.AnnotationSupport.findAnnotatedFields;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.config.Ignite2ConfigurationUtils;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.platform.commons.support.ModifierSupport;
+
+/** BasePersistentTestContext. */
+public class BasePersistentTestContext implements BeforeEachCallback, AfterEachCallback {
+
+ /** recreateNodeContexes. */
+ public static List<MigrationKernalContext> recreateNodeContexes() throws IOException, IgniteCheckedException {
+ IgniteConfiguration igniteCfg = Ignite2ConfigurationUtils.loadIgnite2Configuration(
+ FullSampleCluster.CLUSTER_CFG_PATH.toFile(), false);
+
+ assumeTrue(Files.exists(FullSampleCluster.TEST_CLUSTER_PATH) && Files.isDirectory(FullSampleCluster.TEST_CLUSTER_PATH),
+ "The test-cluster must be recreated before running the tests.\n"
+ + "Please read the tools/sample-cluster-generator/README.md for instructions on how to recreate the test-cluster.\n"
+ + "After running the tool, the cluster workDir should be placed under resources/sample-clusters/test-cluster");
+
+ var cfg = new IgniteConfiguration()
+ .setWorkDirectory(FullSampleCluster.TEST_CLUSTER_PATH.toString())
+ .setDataStorageConfiguration(new DataStorageConfiguration());
+
+ List<Ignite2PersistenceTools.NodeFolderDescriptor> nodeCandidates = Ignite2PersistenceTools.nodeFolderCandidates(cfg);
+ List<MigrationKernalContext> ret = new ArrayList<>(nodeCandidates.size());
+ for (Ignite2PersistenceTools.NodeFolderDescriptor candidate : nodeCandidates) {
+ MigrationKernalContext ctx = new MigrationKernalContext(igniteCfg, candidate.subFolderFile(), candidate.consistentId());
+
+ ret.add(ctx);
+ }
+
+ return ret;
+ }
+
+ private static boolean isCompatible(Field field) {
+ // Should raise an exception if not the correct type.
+ return Arrays.stream(field.getAnnotation(ExtendWith.class).value())
+ .anyMatch(klass -> klass == BasePersistentTestContext.class) && ModifierSupport.isNotStatic(field);
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) throws Exception {
+ var klass = context.getRequiredTestClass();
+ var instance = context.getRequiredTestInstance();
+ for (var field : findAnnotatedFields(klass, ExtendWith.class, BasePersistentTestContext::isCompatible)) {
+ var nodeContexes = recreateNodeContexes();
+ FieldUtils.writeField(field, instance, nodeContexes, true);
+ }
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) throws Exception {
+ var klass = context.getRequiredTestClass();
+ var instance = context.getRequiredTestInstance();
+ for (var field : findAnnotatedFields(klass, ExtendWith.class, BasePersistentTestContext::isCompatible)) {
+ var nodeContexts = (List<MigrationKernalContext>) FieldUtils.readField(field, instance, true);
+ for (MigrationKernalContext ctx : nodeContexts) {
+ ctx.stop();
+ }
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/CacheDdlCreationFromWorkDirTest.java b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/CacheDdlCreationFromWorkDirTest.java
new file mode 100644
index 00000000000..3855517c154
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/CacheDdlCreationFromWorkDirTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.migrationtools.sql.SqlDdlGenerator;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/** CacheDdlCreationFromWorkDirTest. */
+@DisabledIfSystemProperty(
+ named = "tests.containers.support",
+ matches = "false",
+ disabledReason = "Lack of support in TeamCity for testcontainers")
+@ExtendWith(FullSampleCluster.class)
+public class CacheDdlCreationFromWorkDirTest {
+ private static MigrationKernalContext NODE_CTX;
+
+ @BeforeAll
+ static void initNodeCtx() throws IOException, IgniteCheckedException {
+ NODE_CTX = BasePersistentTestContext.recreateNodeContexes().get(0);
+ NODE_CTX.start();
+ }
+
+ @AfterAll
+ static void closeNodeCtx() throws IgniteCheckedException {
+ if (NODE_CTX != null) {
+ NODE_CTX.stop();
+ }
+ }
+
+ private static String generateSqlDdlForCache(String cacheName) {
+ var cacheDescriptor = NODE_CTX.cache().cacheDescriptor(cacheName);
+ var cacheCfg = cacheDescriptor.cacheConfiguration();
+
+ SqlDdlGenerator gen = new SqlDdlGenerator();
+ var tableDef = gen.generateTableDefinition(cacheCfg);
+ return SqlDdlGenerator.createDdlQuery(tableDef);
+ }
+
+ @Test
+ void testCreateDdlForMySimpleCache() {
+ var sqlStr = generateSqlDdlForCache("MySimpleMap");
+ assertThat(sqlStr)
+ .startsWith("CREATE TABLE PUBLIC.\"MySimpleMap\"")
+ .contains("PRIMARY KEY (ID)")
+ .contains("ID VARCHAR")
+ .contains("VAL INT");
+ }
+
+ @Test
+ void testCreateDdlForMyPersonPojoCache() {
+ var sqlStr = generateSqlDdlForCache("MyPersonPojoCache");
+ assertThat(sqlStr)
+ .startsWith("CREATE TABLE PUBLIC.\"MyPersonPojoCache\"")
+ .contains("KEY INT")
+ .contains("PRIMARY KEY (KEY)")
+ .contains("ID BIGINT")
+ .contains("ORGID BIGINT")
+ .contains("FIRSTNAME VARCHAR")
+ .contains("LASTNAME VARCHAR")
+ .contains("RESUME VARCHAR")
+ .contains("SALARY DOUBLE");
+ }
+
+ @Test
+ void testCreateDdlForMyOrgPojoCache() {
+ var sqlStr = generateSqlDdlForCache("MyOrganizations");
+ assertThat(sqlStr)
+ .startsWith("CREATE TABLE PUBLIC.\"MyOrganizations\"")
+ .contains("KEY BIGINT")
+ .contains("PRIMARY KEY (KEY)")
+ .contains("ID BIGINT")
+ .contains("NAME VARCHAR");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"MyIntArrCache", "MyListArrCache"})
+ void testCollectionsCaches(String cacheName) {
+ var sqlStr = generateSqlDdlForCache(cacheName);
+ assertThat(sqlStr)
+ .startsWith("CREATE TABLE PUBLIC.\"" + cacheName + "\"")
+ .contains("PRIMARY KEY (ID)")
+ .contains("ID INT")
+ .contains("VAL VARBINARY(1024)");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"MyBinaryPersonPojoCache", "MyBinaryOrganizationCache", "MyBinaryTestCache"})
+ void testCreateDdlForBinaryCache(String cacheName) {
+ var sqlStr = generateSqlDdlForCache(cacheName);
+ assertThat(sqlStr)
+ .startsWith("CREATE TABLE PUBLIC.\"" + cacheName + "\"")
+ .contains("PRIMARY KEY (ID)")
+ .contains("ID VARBINARY(1024)")
+ .contains("VAL VARBINARY(1024)");
+ }
+}
diff --git a/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/JsonSerializersTests.java b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/JsonSerializersTests.java
new file mode 100644
index 00000000000..bb92e4d2391
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/JsonSerializersTests.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Flow;
+import org.apache.commons.collections4.IteratorUtils;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.examples.model.Organization;
+import org.apache.ignite.internal.binary.BinaryObjectImpl;
+import org.apache.ignite.migrationtools.persistence.mappers.AbstractSchemaColumnsProcessor;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.apache.ignite.migrationtools.tests.e2e.impl.MyOrganizationsCacheTest;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** JsonSerializersTests. */
+@DisabledIfSystemProperty(
+ named = "tests.containers.support",
+ matches = "false",
+ disabledReason = "Lack of support in TeamCity for testcontainers")
+@ExtendWith(FullSampleCluster.class)
+public class JsonSerializersTests {
+ private static final Logger LOGGER = LoggerFactory.getLogger(JsonSerializersTests.class);
+
+ @ExtendWith(BasePersistentTestContext.class)
+ private List<MigrationKernalContext> nodeContexts;
+
+ private List<MigrationKernalContext> firstNode;
+
+ private final ObjectMapper mapper = AbstractSchemaColumnsProcessor.createMapper();
+
+ @BeforeEach
+ void startContexts() throws IgniteCheckedException {
+ firstNode = List.of(nodeContexts.get(0));
+ for (MigrationKernalContext ctx : firstNode) {
+ ctx.start();
+ }
+ // Stop is done automatically.
+ }
+
+ @Test
+ void loadBinaryCacheObjectAndSerialize() throws IgniteCheckedException, JsonProcessingException {
+ var cacheTest = new MyOrganizationsCacheTest();
+ List<String> expectedFieldNames = List.of("addr", "type", "lastUpdated");
+
+ MutableObject<Throwable> errorRef = new MutableObject<>(null);
+ Map<Integer, String> jsonValues = new HashMap<>();
+ Ignite2PersistentCacheTools.publishCacheCursor(firstNode, cacheTest.cacheConfiguration().getName(), (publisher, partId) -> {
+ if (jsonValues.size() >= 10 || errorRef.getValue() != null) {
+ return Optional.empty();
+ }
+
+ CompletableFuture<Void> f = new CompletableFuture<>();
+ publisher.subscribe(new Flow.Subscriber<>() {
+ private Flow.Subscription subscription;
+
+ @Override
+ public void onSubscribe(Flow.Subscription subscription) {
+ this.subscription = subscription;
+ this.subscription.request(1);
+ }
+
+ @Override
+ public void onNext(Map.Entry<Object, Object> item) {
+ var tmp = new AbstractSchemaColumnsProcessor.WrapperClass((BinaryObjectImpl) item.getValue(), expectedFieldNames);
+
+ try {
+ String jsonStr = mapper.writeValueAsString(tmp);
+ jsonValues.put(Math.toIntExact((Long) item.getKey()), jsonStr);
+ this.subscription.request(1);
+ } catch (JsonProcessingException e) {
+ onError(e);
+ }
+ }
+
+ @Override
+ public void onError(Throwable throwable) {
+ LOGGER.error("Error on subscriber", throwable);
+ errorRef.setValue(throwable);
+ subscription.cancel();
+ f.complete(null);
+ }
+
+ @Override
+ public void onComplete() {
+ f.complete(null);
+ }
+ });
+
+ return Optional.of(f);
+ });
+
+ assertThat(errorRef.getValue()).isNull();
+
+ var simpleMapper = new ObjectMapper()
+ .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
+ for (var e : jsonValues.entrySet()) {
+ String jsonStr = e.getValue();
+ var ex = cacheTest.supplyExample(e.getKey()).getValue();
+
+ JsonNode node = simpleMapper.readTree(jsonStr);
+
+ assertThat(IteratorUtils.toList(node.fieldNames()))
+ .containsExactlyInAnyOrderElementsOf(expectedFieldNames);
+
+ var actual = simpleMapper.readValue(jsonStr, Organization.class);
+
+ assertThat(actual)
+ .usingRecursiveComparison()
+ .ignoringFields("id", "name")
+ .isEqualTo(ex);
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/ListCachesTest.java b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/ListCachesTest.java
new file mode 100644
index 00000000000..5bd63632aaf
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/ListCachesTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/** ListCachesTest. */
+@DisabledIfSystemProperty(
+ named = "tests.containers.support",
+ matches = "false",
+ disabledReason = "Lack of support in TeamCity for testcontainers")
+@ExtendWith(FullSampleCluster.class)
+public class ListCachesTest {
+
+ @ExtendWith(BasePersistentTestContext.class)
+ private List<MigrationKernalContext> nodeContexts;
+
+ @BeforeEach
+ void startContexts() throws IgniteCheckedException {
+ for (MigrationKernalContext ctx : nodeContexts) {
+ ctx.start();
+ }
+ // Stop is done automatically.
+ }
+
+ @Test
+ void checkCachesAreInTheSampleCluster() throws IgniteCheckedException {
+ var foundCaches = Ignite2PersistentCacheTools.persistentCaches(nodeContexts);
+ assertThat(foundCaches).hasSize(8);
+ // Value extract from the seeddata-container.log
+ }
+
+}
diff --git a/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/PdsNodeDiscoveryTests.java b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/PdsNodeDiscoveryTests.java
new file mode 100644
index 00000000000..e5b9541109c
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/integrationTest/java/org/apache/ignite/migrationtools/persistence/PdsNodeDiscoveryTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.migrationtools.tests.clusters.FullSampleCluster;
+import org.apache.ignite.migrationtools.tests.containers.Ignite2ClusterContainer;
+import org.assertj.core.groups.Tuple;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/** PdsNodeDiscoveryTests. */
+@ExtendWith(FullSampleCluster.class)
+public class PdsNodeDiscoveryTests {
+
+ @Test
+ void checkSampleClusterNodes() {
+ var cfg = new IgniteConfiguration()
+ .setWorkDirectory(FullSampleCluster.TEST_CLUSTER_PATH.toString())
+ .setDataStorageConfiguration(new DataStorageConfiguration());
+ var candidates = Ignite2PersistenceTools.nodeFolderCandidates(cfg);
+ assertThat(candidates)
+ .extracting(e -> e.consistentId().toString(), e -> e.subFolderFile().getName())
+ .containsExactlyInAnyOrder(
+ Tuple.tuple("7b880b69-8a9e-4b84-b555-250d365e2e67", "7b880b69_8a9e_4b84_b555_250d365e2e67"),
+ Tuple.tuple("ad26bff6-5ff5-49f1-9a61-425a827953ed", "ad26bff6_5ff5_49f1_9a61_425a827953ed"),
+ Tuple.tuple("c1099d16-e7d7-49f4-925c-53329286c444", "c1099d16_e7d7_49f4_925c_53329286c444")
+ );
+ }
+
+ @Nested
+ class NonUuidConsistentIds {
+ private final Path clusterFolder = FullSampleCluster.SAMPLE_CLUSTERS_PATH.resolve(this.getClass().getSimpleName());
+
+ private final List<String> nodeConsistentIds = List.of("node1", "node2", "node3");
+
+ @BeforeEach
+ void setupCluster() {
+ if (Files.notExists(clusterFolder)) {
+ var cluster = new Ignite2ClusterContainer(
+ FullSampleCluster.CLUSTER_CFG_PATH,
+ clusterFolder,
+ nodeConsistentIds);
+
+ try {
+ cluster.start();
+ } finally {
+ cluster.doStop(false);
+ }
+ }
+ }
+
+ @Test
+ void checkClusterNodeNames() {
+ var cfg = new IgniteConfiguration()
+ .setWorkDirectory(clusterFolder.toString())
+ .setDataStorageConfiguration(new DataStorageConfiguration());
+
+ List<Ignite2PersistenceTools.NodeFolderDescriptor> candidates = Ignite2PersistenceTools.nodeFolderCandidates(cfg);
+
+ assertThat(candidates)
+ .extracting(Ignite2PersistenceTools.NodeFolderDescriptor::consistentId)
+ .containsExactlyInAnyOrderElementsOf(nodeConsistentIds);
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/Ignite2PersistenceTools.java b/migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/Ignite2PersistenceTools.java
new file mode 100644
index 00000000000..61851140354
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/Ignite2PersistenceTools.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Utility class to handle Ignite 2 persistence work dirs. */
+public class Ignite2PersistenceTools {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Ignite2PersistenceTools.class);
+
+ // UUIDs as they are masked by org.apache.ignite.internal.util.IgniteUtils#maskForFileName
+ private static final Pattern MASKED_UUID_PATTERN =
+ Pattern.compile("[a-fA-F0-9]{8}_[a-fA-F0-9]{4}_[a-fA-F0-9]{4}_[a-fA-F0-9]{4}_[a-fA-F0-9]{12}");
+
+ private static final Method FIND_PERSISTENT_STORAGE_PATH_METHOD;
+
+ private static final Method FIND_NODE_FOLDERS_METHOD;
+
+ private static final FileFilter DB_SUBFOLDERS_OLD_STYLE_FILTER;
+
+ static {
+ try {
+ FIND_PERSISTENT_STORAGE_PATH_METHOD =
+ PdsFolderResolver.class.getDeclaredMethod("resolvePersistentStoreBasePath", boolean.class);
+ FIND_PERSISTENT_STORAGE_PATH_METHOD.setAccessible(true);
+ FIND_NODE_FOLDERS_METHOD = PdsFolderResolver.class.getDeclaredMethod("getNodeIndexSortedCandidates", File.class);
+ FIND_NODE_FOLDERS_METHOD.setAccessible(true);
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static {
+ try {
+ Field field = PdsFolderResolver.class.getDeclaredField("DB_SUBFOLDERS_OLD_STYLE_FILTER");
+ field.setAccessible(true);
+ DB_SUBFOLDERS_OLD_STYLE_FILTER = (FileFilter) field.get(null);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Compute a list of {@link NodeFolderDescriptor}s available for the provided Ignite 2 cluster configuration.
+ *
+ * @param cfg The Ignite 2 cluster configuration.
+ * @return List of node folders.
+ */
+ public static List<NodeFolderDescriptor> nodeFolderCandidates(IgniteConfiguration cfg) {
+ if (cfg.getWorkDirectory() == null) {
+ throw new IllegalArgumentException("WorkDirectory must not be null");
+ }
+
+ if (cfg.getDataStorageConfiguration() == null) {
+ throw new IllegalArgumentException("DataStorageConfiguration must not be null");
+ }
+
+ File dbFolderFile;
+ List<PdsFolderResolver.FolderCandidate> candidates;
+
+ try {
+ var folderResolver = new PdsFolderResolver<>(cfg, null, null, null);
+ dbFolderFile = (File) FIND_PERSISTENT_STORAGE_PATH_METHOD.invoke(folderResolver, false);
+ candidates = (List<PdsFolderResolver.FolderCandidate>) FIND_NODE_FOLDERS_METHOD.invoke(folderResolver, dbFolderFile);
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+
+ // Fallback trying to find folders using the old naming convention. Adapted from PDSResolver
+ if (candidates == null || candidates.isEmpty()) {
+ final File[] oldStyleFolders = dbFolderFile.listFiles(DB_SUBFOLDERS_OLD_STYLE_FILTER);
+
+ if (oldStyleFolders != null && oldStyleFolders.length != 0) {
+ List<NodeFolderDescriptor> ret = new ArrayList<>(oldStyleFolders.length);
+
+ for (File folder : oldStyleFolders) {
+ String folderName = folder.getName();
+ Matcher matcher = MASKED_UUID_PATTERN.matcher(folderName);
+ if (matcher.matches()) {
+ String uuidStr = folderName.replace('_', '-');
+ var uuid = UUID.fromString(uuidStr);
+ ret.add(new NodeFolderDescriptor(folder, uuid));
+ } else {
+ LOGGER.warn("Could not capture consistentId (UUID) from node folder, falling back to String: {}", folder.getPath());
+ ret.add(new NodeFolderDescriptor(folder, folderName));
+ }
+ }
+
+ return ret;
+ } else {
+ return Collections.emptyList();
+ }
+ } else {
+ return candidates.stream()
+ .map(folderCandidate -> new NodeFolderDescriptor(folderCandidate.subFolderFile(), folderCandidate.uuid()))
+ .collect(Collectors.toList());
+ }
+ }
+
+ /**
+ * Simple descriptor of discovered node folders.
+ */
+ public static class NodeFolderDescriptor {
+ /** Absolute file path pointing to DB subfolder within DB storage root folder. */
+ private final File subFolderFile;
+
+ /** ConsistentId of the Node. */
+ private final Serializable consistentId;
+
+ public NodeFolderDescriptor(File subFolderFile, Serializable consistentId) {
+ this.subFolderFile = subFolderFile;
+ this.consistentId = consistentId;
+ }
+
+ /**
+ * consistentId.
+ *
+ * @return Uuid contained in file name, is to be set as consistent ID.
+ */
+ public Serializable consistentId() {
+ return consistentId;
+ }
+
+ /**
+ * subFolderFile.
+ *
+ * @return Absolute file path pointing to DB subfolder within DB storage root folder.
+ */
+ public File subFolderFile() {
+ return subFolderFile;
+ }
+ }
+}
diff --git a/migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/Ignite2PersistentCacheTools.java b/migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/Ignite2PersistentCacheTools.java
new file mode 100644
index 00000000000..e55fc5fe663
--- /dev/null
+++ b/migration-tools/modules/migration-tools-persistence/src/main/java/org/apache/ignite/migrationtools/persistence/Ignite2PersistentCacheTools.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrationtools.persistence;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Flow;
+import java.util.function.BiFunction;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.cache.CacheType;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.util.lang.GridCursor;
+import org.apache.ignite.migrationtools.persistence.mappers.CacheDataRowProcessor;
+import org.apache.ignite.migrationtools.persistence.utils.pubsub.StreamerPublisher;
+import org.apache.ignite.migrationtools.sql.SqlDdlGenerator;
+import org.apache.ignite.migrationtools.tablemanagement.SchemaUtils;
+import org.apache.ignite.migrationtools.types.converters.StaticTypeConverterFactory;
+import org.apache.ignite.migrationtools.types.converters.TypeConverterFactory;
+import org.apache.ignite3.client.IgniteClient;
+import org.apache.ignite3.internal.client.table.ClientSchema;
+import org.apache.ignite3.internal.client.table.ClientTable;
+import org.apache.ignite3.lang.util.IgniteNameUtils;
+import org.apache.ignite3.table.DataStreamerItem;
+import org.apache.ignite3.table.Tuple;
+import org.jetbrains.annotations.Nullable;
+
+/** Utility methods to interact with Ignite 2 persistent caches. */
+public class Ignite2PersistentCacheTools {
+
+ /**
+ * List the persistent caches in the provided nodes.
+ *
+ * @param nodes The cluster nodes.
+ * @return Set of cacheId cacheName pairs.
+ * @throws IgniteCheckedException in case of error.
+ */
+ public static Set<Pair<Integer, String>> persistentCaches(List<MigrationKernalContext> nodes) throws IgniteCheckedException {
+ Set<Pair<Integer, String>> cacheIds = new HashSet<>();
+ for (MigrationKernalContext ctx : nodes) {
+ var cacheCtx = ctx.cache();
+ if (cacheCtx instanceof MigrationCacheProcessor) {
+ ((MigrationCacheProcessor) cacheCtx).loadAllDescriptors();
+ }
+
+ for (var descr : ctx.cache().persistentCaches()) {
+ if (CacheType.USER.equals(descr.cacheType())) {
+ cacheIds.add(Pair.of(descr.cacheId(), descr.cacheName()));
+ }
+ }
+ }
+ return cacheIds;
+ }
+
+ /**
+ * Publishes a cache cursor on the given nodes.
+ *
+ * @param nodes Cluster nodes.
+ * @param cacheName Cache name.
+ * @param partitionStreamFactory Function that allows the caller to react when each partition cursor is opened.
+ * @throws IgniteCheckedException in case of error.
+ */
+ public static void publishCacheCursor(
+ List<MigrationKernalContext> nodes,
+ String cacheName,
+ BiFunction<Flow.Publisher<Map.Entry<Object, Object>>, Integer, Optional<CompletableFuture<Void>>> partitionStreamFactory
+ ) throws IgniteCheckedException {
+ for (MigrationKernalContext nodeCtx : nodes) {
+ DynamicCacheDescriptor cacheDescriptor = nodeCtx.cache().cacheDescriptor(cacheName);
+ publishCacheCursorAtNode(nodeCtx, cacheDescriptor, partitionStreamFactory);
+ }
+ }
+
+ /**
+ * Publishes a cache cursor on the given node.
+ *
+ * @param nodeCtx Cluster node context.
+ * @param cacheDescriptor Cache descriptor.
+ * @param partitionStreamFactory Function that allows the caller to react when each partition cursor is opened.
+ * @throws IgniteCheckedException in case of error.
+ */
+ public static void publishCacheCursorAtNode(
+ MigrationKernalContext nodeCtx,
+ DynamicCacheDescriptor cacheDescriptor,
+ BiFunction<Flow.Publisher<Map.Entry<Object, Object>>, Integer, Optional<CompletableFuture<Void>>> partitionStreamFactory
+ ) throws IgniteCheckedException {
+ var cacheProcessor = nodeCtx.cache();
+ if (cacheProcessor instanceof MigrationCacheProcessor) {
+ ((MigrationCacheProcessor) cacheProcessor).startCache(cacheDescriptor);
+ }
+ int cacheId = cacheDescriptor.cacheId();
+ var gridCacheSharedContext = cacheProcessor.context();
+ var cacheContext = gridCacheSharedContext.cacheContext(cacheId);
+ var offHeap = cacheContext.offheap();
+ var cacheObjectCtx = cacheContext.cacheObjectContext();
+
+ for (IgniteCacheOffheapManager.CacheDataStore cs : offHeap.cacheDataStores()) {
+ StreamerPublisher<CacheDataRow> cacheTuplesPublisher = new StreamerPublisher<>();
+
+ CacheDataRowProcessor rowProc = new CacheDataRowProcessor(cacheObjectCtx);
+ cacheTuplesPublisher.subscribe(rowProc);
+
+ var streamJobOpt = partitionStreamFactory.apply(rowProc, cs.partId());
+ if (streamJobOpt.isPresent()) {
+ CompletableFuture<Void> streamJob = streamJobOpt.get();
+ try {
+ // TODO: GG-40802 Allow more control on the converters side.
+ GridCursor<? extends CacheDataRow> cu = cs.cursor();
+
+ while (cu.next()) {
+ CacheDataRow row = cu.get();
+ // Currently, I don't know how to check this more efficiently.
+ // Skips other caches in the same grp. There might be a better way to do this.
+ // Caches outside groups may have the cacheId set to 0
+ if (row.cacheId() != 0 && row.cacheId() != cacheId) {
+ continue;
+ }
+
+ try {
+ boolean success = cacheTuplesPublisher.offer(row);
+ if (!success) {
+ throw new IgniteCheckedException("Failed to offer element to publisher");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IgniteCheckedException("Interrupted publishing tuple");
+ }
+ }
+ } catch (IgniteCheckedException ex) {
+ cacheTuplesPublisher.closeExceptionally(ex);
+ throw ex;
+ } finally {
+ cacheTuplesPublisher.close();
+ streamJob.join();
+ }
+ }
+ }
+ }
+
+ /**
+ * Wrapper method that migrates a given cache.
+ *
+ * @param client Adapted client.
+ * @param sqlGenerator SQL DDL Generator.
+ * @param nodeContexts Node contexts.
+ * @param cacheName Cache name.
+ * @param columnsProcessorFactory Column processor factory.
+ * @throws IgniteCheckedException in case of error.
+ */
+ public static void migrateCache(
... 5772 lines suppressed ...