You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by "panrobin (via GitHub)" <gi...@apache.org> on 2026/08/12 08:23:34 UTC
[I] [Bug] [SqlServer-CDC] Filter transform include_fields not working with generate_sink_sql=true in SqlServer-CDC multi-table scenario [seatunnel]
panrobin opened a new issue, #11774:
URL: https://github.com/apache/seatunnel/issues/11774
### Search before asking
- [x] I had searched in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found no similar issues.
### What happened
## Environment
- SeaTunnel Version: 2.3.13
- Engine: Zeta
- Source: SqlServer-CDC
- Sink: JDBC (MySQL)
## Bug Description
When using SqlServer-CDC source with Filter transform (include_fields)
and JDBC sink (generate_sink_sql=true), the generated INSERT SQL contains
fields that are NOT in the Filter's include_fields list.
The Filter transform does not correctly strip extra fields before
generate_sink_sql builds the PreparedStatement.
## Steps to Reproduce
1. Configure SqlServer-CDC source with a table that has extra columns
(e.g., BillMaker) not present in MySQL target table
2. Add Filter transform with include_fields that excludes BillMaker
3. Configure JDBC sink with generate_sink_sql=true
4. Run the job
## Expected Behavior
The generated SQL should only contain fields listed in include_fields,
BillMaker should be excluded.
## Actual Behavior
The generated SQL includes BillMaker, causing:
java.sql.SQLSyntaxErrorException: Unknown column 'BillMaker' in 'field list'
### SeaTunnel Version
SeaTunnel Version: 2.3.13
### SeaTunnel Config
```conf
env {
parallelism = 1
job.mode = "STREAMING"
checkpoint.interval = 30000
checkpoint.timeout = 30000
job.retry.times = 3
job.retry.interval.seconds = 3
}
source {
SqlServer-CDC {
plugin_output = "sqlserver_af"
username = "cdcuser"
password = "******"
url = "jdbc:sqlserver://**.**.**.**:1433;databaseName=QG_SC_CIIC_AFOne_Data;encrypt=false;trustServerCertificate=true"
database-names = [
"QG_SC_CIIC_AFOne_Data",
"QG_SC_CIIC_AFONE_MasterData"
]
table-names = [
"QG_SC_CIIC_AFOne_Data.dbo.SYS_User",
"QG_SC_CIIC_AFOne_Data.dbo.SYS_OrgUnit",
"QG_SC_CIIC_AFOne_Data.dbo.SAL_Company",
"QG_SC_CIIC_AFOne_Data.dbo.SAL_ComAdmin",
"QG_SC_CIIC_AFOne_Data.dbo.SAL_Customer",
"QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Employee",
"QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Company",
"QG_SC_CIIC_AFONE_MasterData.dbo.Com_Company"
]
table-names-config = [
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SYS_User", primaryKeys = ["UserId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SYS_OrgUnit", primaryKeys = ["OrgUnitId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SAL_Company", primaryKeys = ["CompanyId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SAL_ComAdmin", primaryKeys = ["ComAdminId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SAL_Customer", primaryKeys = ["CustomerId"] },
{ table = "QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Employee",primaryKeys = ["ID"] },
{ table = "QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Company", primaryKeys = ["EmpCompanyID"] },
{ table = "QG_SC_CIIC_AFONE_MasterData.dbo.Com_Company", primaryKeys = ["CompanyID"] }
]
startup.mode = "initial"
exactly_once = true
snapshot.split.size = 8096
fetch.size = 1024
poll.interval.ms = 1000
debezium {
database.encrypt = false
database.trustServerCertificate = true
}
}
}
transform {
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_sys_user"
table_match_regex = ".*QG_SC_CIIC_AFOne_Data\\.dbo\\.SYS_User"
include_fields = [
"UserId","Name","EnglishName","LoginName","EmployeeNumber","ADAccount","Password",
"UserType","Email","Mobile","ExtCode","Status","Description","OrgUnitID","OrgUnitCode",
"MainPositionId","ServiceCenterId","ServiceCenterCode","Avatar","PositionType","LoginTime",
"LastLoginTime","ErrorCount","LastErrorTime","IsLock","LockTime","CreatedBy","CreatedTime",
"LastUpdatedBy","LastUpdatedTime","IsDelete","RowVersion","OrgUintAuthority","NCPreparedCode",
"WICPUserId","WICPUserType","TenantId","OperateAuthType","City","LastUserSecrecyUpdateTime",
"LastLoginStatus","LastLockTime","LoginNameByGroup"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_sys_orgunit"
table_match_regex = ".*QG_SC_CIIC_AFOne_Data\\.dbo\\.SYS_OrgUnit"
include_fields = [
"OrgUnitId","OrgUnitCode","OrgUnitName","ShortName","OrgUnitType","ParentOrgUnitID",
"CodePath","Deep","Description","Sort","Disabled","CreatedBy","CreatedTime",
"LastUpdatedBy","LastUpdatedTime","IsDelete","LegalEntityId","RowVersion"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_sal_company"
table_match_regex = ".*QG_SC_CIIC_AFOne_Data\\.dbo\\.SAL_Company"
include_fields = [
"CompanyId","CustomerId","ContractCategoryId","ContractSubCategoryId","LegalEntityId",
"CompanyNo","AfCompanyNo","CustomerName","CompanyNameSuffix","CompanyName","Remark",
"Status","Disabled","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","IsDelete",
"RowVersion","TeamOrgId","MainCustomerId","ComAdminGroupId","ServiceTypeNotes","ShWorkType",
"ShWorkName","ShGjjAccount","ShSbAccount","IstEmploymentOnly","UniformSocialCreditCode",
"EntrustTypeNotes","IsESign","IsSubpackage","SubpackageType","CostCenterID","CostCenterCode",
"CostCenterName","ContractLegalEntityId","DelegateLegalEntityId","StampedEntityCompany",
"ComInvoiceCategoryID","IsTax","TaxRate","IsVATSUTTax","VATSUTTaxRate","EntrustType",
"CompanyType","TemplateId"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_sal_comadmin"
table_match_regex = ".*QG_SC_CIIC_AFOne_Data\\.dbo\\.SAL_ComAdmin"
include_fields = [
"ComAdminId","ComAdminCode","ComAdminName","ComAdminType","Description","OtherName",
"IsFiveHundred","Source","Status","CountryCode","CountryName","FamousProduct","OwnerUserId",
"Remark","Disabled","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","IsDelete",
"RowVersion","IsCertified","EmployeeType","ComAdminGroupName","SiteCode"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_sal_customer"
table_match_regex = ".*QG_SC_CIIC_AFOne_Data\\.dbo\\.SAL_Customer"
include_fields = [
"CustomerId","ComAdminId","CustomerNo","CustomerName","IsMain","CustomerStatus","CustomerType",
"UniformSocialCreditCode","BusinessValidDate","LicenseType","LicenseCode","RegisteredCapital",
"LegalPerson","AddressProvence","AddressCity","Address","Telephone","Fax","IndustryCategory",
"Industry","Country","Nature","Industry1","CustomerClass","QualificationType","BpoRemark",
"DepositInfo","Source","SourceVendorName","Remark","EndRemark","EndBy","EndOperateIP",
"Disabled","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","IsDelete","RowVersion",
"ConfirmationDate","IsCentral","CentralCompanyName","CentralManageLevel","firstSignDate",
"IsCompetitor","IsBidding","BiddingProjectNo","VendorOperationMode","VendorOperationModeName",
"MDMCode","MDMMasterID","MDMhmdbz","CNPlatformCode","BindAdminType","IsMainCustomer","MainCustomerID"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_emp_employee"
table_match_regex = ".*QG_SC_CIIC_AFONE_MasterData\\.dbo\\.Emp_Employee"
include_fields = [
"ID","EmployeeID","EmployeeName","IDNumber","IDType","Mobile","Birthdate","Gender",
"MaritalStatus","Education","Email","WechatNo","WechatNick","Telephone","ZipCode","Address",
"Province","City","Area","Street","Way","Village","Number","Room","ResidentAddress",
"ResidentArea","ResidentStreet","ResidentWay","ResidentVillage","ResidentNo","ResidentRoom",
"ResidenZipCode","Nationality","Politics","Nation","PhdType","PhdStatus","SeniorEmpNo",
"ChildrenNum","IsResidence","ResidenceType","ResidenceNo","BeforeName","SocialInfo","FundInfo",
"EmpSource","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","RowVersion",
"ResidentProvince","ResidentCity","CertificateStartDate","CertificateEndDate","Major",
"GraduateSchool","GraduateTime","EmployeeTitle","RCGWId","EnglishName","PinYin",
"IsIndustrialInjury","IsVeteran","IsDisability","IsPayroll"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_emp_company"
table_match_regex = ".*QG_SC_CIIC_AFONE_MasterData\\.dbo\\.Emp_Company"
include_fields = [
"EmpCompanyID","EmployeeID","AFCompanyID","CustomerNo","ComAdminCode","FirstDate",
"EmpAttribute","IsTerminated","TerminationDate","TerminationReason","TerminalBy",
"TerminationOperateDate","SocialEndDate","FundEndDate","InsuranceEndDate","Position",
"JobNumber","DeptName","IsApplyCard","Health","Skill","JobProfession","Post","WorkPlace",
"Brand","WorkArea","EmpFullName","SendCardDate","HandelCardDate","Is_NoCard","CostumesMonth",
"CostumesBalance","InDate","ProbationStart","ProbationEnd","Retire55","Rehire","RehireAge",
"Remark","ArTer","ArTerFeedback","ArTerOutDate","SocialStatus","SocialDate","IsSocialHandle",
"SocialNotice","GroupID","DeductGroupID","EmploymentType","EmpSource","CreatedBy","CreatedTime",
"LastUpdatedBy","LastUpdatedTime","RowVersion","EmpNature","ExemptionCreditDate",
"EmployeeIDBefore","EmpRemark","IsViewPermission","IsTransfer","TransferDate","PfileTagID",
"PersonNatrue","EmploymentAttribute","SSContractStartDate","PfileTagStartMonth","PfileTagEndMonth",
"TerminationRemark","TerminationProceduresDate","SourceType","CertificationStatus",
"CertificationFailureReason","DaySalary","HourSalary","IsMigrantWorkers","GraduationDate",
"EmployeeType","PackageID","SSAndHFOutgoingRules","SSAccountType","SSAccountName","City",
"ServiceUserID","ProofSignetID","ProofSignetName","RehireDate","IsLabourServices",
"LabourServicesType","IsApply","LabourMaterials","IsNeedSignContract","ContractType",
"EmpUseType","IsPreResign","ZytEmpId","BusEmploymentType"
]
}
Filter {
plugin_input = "sqlserver_af"
plugin_output = "af_com_company"
table_match_regex = ".*QG_SC_CIIC_AFONE_MasterData\\.dbo\\.Com_Company"
include_fields = [
"CompanyID","AFCompanyID","ComAdminId","ComAdminCode","CustomerId","CustomerNo","CompanyName",
"CompanyEnName","ServiceCenterID","ServiceCentre","OrgUnitID","CusManger","EmpCount",
"ServiceTypeID","ServiceType","CustomerProperty","CompanyNature","IsTerminated","TerminationDate",
"TerminationReason","TerminationOpDate","Remark","EsRemark","CreditPeriod","MgtCreditPeriod",
"PayrollCreditPeriod","IsUpload","IsEHR","CsOperator","IsExpenseSuspend","ReimbursementCard",
"SalaryCard","IsSalary","StatementArea","LegalEntityID","OrgCode","IsIdtEmployment","Counsellor",
"EntryDepart","Industry","TaxDeclareMonth","PayrollDeclareMonth","TaxRulePeriod","CompanyType",
"CompanyAddress","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","RowVersion",
"IstEmploymentOnly","CusTeam","SocialSecurityMonth","SocialSecurityDay","SalaryMonth","SalaryDay",
"ServiceFeeMonth","ServiceFeeDay","ServiceTypeNotes","InvoiceWhereabouts","PrincipalPerson",
"IsIntel","ShWorkType","IsContract","SalaryType","ShWorkName","CanInvoice","EntrustTypeNotes",
"BillScope","IsESign","ESignStartDate","ESignEndDate","IsSubpackage","SubpackageType",
"isAutoCreateHealthTask","IsCallCenter","IsSyncESign","LegalSubject","ContractLegalEntityId",
"DelegateLegalEntityId","IsAgreementChangedCreateTask","StampedEntityCompany","IsYDDLKH",
"IsSelfManual","SecTerminationReason","TerTerminationReason","SFSSTYZCYX","ComInvoiceCategoryID",
"AffiliatedCompanyName","SyncESignType"
]
}
}
sink {
Jdbc {
plugin_input = "af_sys_user"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "********"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sys_orgunit"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sal_company"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "*******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sal_comadmin"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sal_customer"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_emp_employee"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_emp_company"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_com_company"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
}
```
### Running Command
```shell
nohup /home/apache-seatunnel-2.3.13/bin/seatunnel.sh --config /home/apache-seatunnel-2.3.13/job/sc_test.conf -m local > /home/apache-seatunnel-2.3.13/job/cdc_sync_data.log 2>&1 &
```
### Error Exception
```log
2026-08-12 13:39:22,656 INFO [.e.FieldNamedPreparedStatement] [st-multi-table-sink-writer-1] - PrepareStatement sql is:
INSERT INTO `ciic_sc`.`SAL_Company` (`CompanyId`, `CustomerId`, `ContractCategoryId`, `ContractSubCategoryId`, `LegalEntityId`, `CompanyNo`, `AfCompanyNo`, `CustomerName`, `CompanyNameSuffix`, `CompanyName`, `Remark`, `Status`, `Disabled`, `CreatedBy`, `CreatedTime`, `LastUpdatedBy`, `LastUpdatedTime`, `IsDelete`, `RowVersion`, `TeamOrgId`, `MainCustomerId`, `ComAdminGroupId`, `ServiceTypeNotes`, `ShWorkType`, `ShWorkName`, `ShGjjAccount`, `ShSbAccount`, `IstEmploymentOnly`, `UniformSocialCreditCode`, `EntrustTypeNotes`, `IsESign`, `IsSubpackage`, `SubpackageType`, `CostCenterID`, `CostCenterCode`, `CostCenterName`, `ContractLegalEntityId`, `DelegateLegalEntityId`, `StampedEntityCompany`, `ComInvoiceCategoryID`, `IsTax`, `TaxRate`, `IsVATSUTTax`, `VATSUTTaxRate`, `EntrustType`, `CompanyType`, `TemplateId`, `BillMaker`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE
KEY UPDATE `CompanyId`=VALUES(`CompanyId`), `CustomerId`=VALUES(`CustomerId`), `ContractCategoryId`=VALUES(`ContractCategoryId`), `ContractSubCategoryId`=VALUES(`ContractSubCategoryId`), `LegalEntityId`=VALUES(`LegalEntityId`), `CompanyNo`=VALUES(`CompanyNo`), `AfCompanyNo`=VALUES(`AfCompanyNo`), `CustomerName`=VALUES(`CustomerName`), `CompanyNameSuffix`=VALUES(`CompanyNameSuffix`), `CompanyName`=VALUES(`CompanyName`), `Remark`=VALUES(`Remark`), `Status`=VALUES(`Status`), `Disabled`=VALUES(`Disabled`), `CreatedBy`=VALUES(`CreatedBy`), `CreatedTime`=VALUES(`CreatedTime`), `LastUpdatedBy`=VALUES(`LastUpdatedBy`), `LastUpdatedTime`=VALUES(`LastUpdatedTime`), `IsDelete`=VALUES(`IsDelete`), `RowVersion`=VALUES(`RowVersion`), `TeamOrgId`=VALUES(`TeamOrgId`), `MainCustomerId`=VALUES(`MainCustomerId`), `ComAdminGroupId`=VALUES(`ComAdminGroupId`), `ServiceTypeNotes`=VALUES(`ServiceTypeNotes`), `ShWorkType`=VALUES(`ShWorkType`), `ShWorkName`=VALUES(`ShWorkName`), `ShGjjAccount`=VALUES(`ShGjj
Account`), `ShSbAccount`=VALUES(`ShSbAccount`), `IstEmploymentOnly`=VALUES(`IstEmploymentOnly`), `UniformSocialCreditCode`=VALUES(`UniformSocialCreditCode`), `EntrustTypeNotes`=VALUES(`EntrustTypeNotes`), `IsESign`=VALUES(`IsESign`), `IsSubpackage`=VALUES(`IsSubpackage`), `SubpackageType`=VALUES(`SubpackageType`), `CostCenterID`=VALUES(`CostCenterID`), `CostCenterCode`=VALUES(`CostCenterCode`), `CostCenterName`=VALUES(`CostCenterName`), `ContractLegalEntityId`=VALUES(`ContractLegalEntityId`), `DelegateLegalEntityId`=VALUES(`DelegateLegalEntityId`), `StampedEntityCompany`=VALUES(`StampedEntityCompany`), `ComInvoiceCategoryID`=VALUES(`ComInvoiceCategoryID`), `IsTax`=VALUES(`IsTax`), `TaxRate`=VALUES(`TaxRate`), `IsVATSUTTax`=VALUES(`IsVATSUTTax`), `VATSUTTaxRate`=VALUES(`VATSUTTaxRate`), `EntrustType`=VALUES(`EntrustType`), `CompanyType`=VALUES(`CompanyType`), `TemplateId`=VALUES(`TemplateId`), `BillMaker`=VALUES(`BillMaker`)
2026-08-12 13:39:22,661 INFO [.e.FieldNamedPreparedStatement] [st-multi-table-sink-writer-1] - PrepareStatement sql is:
DELETE FROM `ciic_sc`.`SAL_Company` WHERE `CompanyId` = ?
2026-08-12 13:39:22,914 ERROR [o.a.s.c.s.j.i.JdbcOutputFormat] [st-multi-table-sink-writer-1] - JDBC executeBatch error, retry times = 0
java.sql.BatchUpdateException: Unknown column 'BillMaker' in 'field list'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_251]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_251]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_251]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_251]
at com.mysql.cj.util.Util.handleNewInstance(Util.java:192) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.util.Util.getInstance(Util.java:167) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.util.Util.getInstance(Util.java:174) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.exceptions.SQLError.createBatchUpdateException(SQLError.java:224) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchWithMultiValuesClause(ClientPreparedStatement.java:718) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchInternal(ClientPreparedStatement.java:409) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.StatementImpl.executeBatch(StatementImpl.java:795) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at org.apache.seatunnel.shade.com.zaxxer.hikari.pool.ProxyStatement.executeBatch(ProxyStatement.java:127) ~[connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.shade.com.zaxxer.hikari.pool.HikariProxyPreparedStatement.executeBatch(HikariProxyPreparedStatement.java) ~[connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.executor.FieldNamedPreparedStatement.executeBatch(FieldNamedPreparedStatement.java:540) ~[connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.executor.SimpleBatchStatementExecutor.executeBatch(SimpleBatchStatementExecutor.java:54) ~[connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.executor.BufferReducedBatchStatementExecutor.executeBatch(BufferReducedBatchStatementExecutor.java:89) ~[connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.JdbcOutputFormat.attemptFlush(JdbcOutputFormat.java:172) ~[connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.JdbcOutputFormat.flush(JdbcOutputFormat.java:136) [connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.internal.JdbcOutputFormat.writeRecord(JdbcOutputFormat.java:106) [connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.sink.JdbcSinkWriter.write(JdbcSinkWriter.java:160) [connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.connectors.seatunnel.jdbc.sink.JdbcSinkWriter.write(JdbcSinkWriter.java:49) [connector-jdbc-2.3.13.jar:2.3.13]
at org.apache.seatunnel.api.sink.multitablesink.MultiTableWriterRunnable.run(MultiTableWriterRunnable.java:67) [seatunnel-starter.jar:2.3.13]
at org.apache.seatunnel.api.tracing.MDCRunnable.run(MDCRunnable.java:43) [seatunnel-starter.jar:2.3.13]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_251]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_251]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_251]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_251]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_251]
Caused by: java.sql.SQLSyntaxErrorException: Unknown column 'BillMaker' in 'field list'
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:120) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:122) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeInternal(ClientPreparedStatement.java:916) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeUpdateInternal(ClientPreparedStatement.java:1061) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeUpdateInternal(ClientPreparedStatement.java:1009) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeLargeUpdate(ClientPreparedStatement.java:1320) ~[mysql-connector-java-8.0.30.jar:8.0.30]
at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchWithMultiValuesClause(ClientPreparedStatement.java:679) ~[mysql-connector-java-8.0.30.jar:8.0.30]
... 19 more
```
### Zeta or Flink or Spark Version
_No response_
### Java or Scala Version
_No response_
### Screenshots
_No response_
### Are you willing to submit PR?
- [ ] Yes I am willing to submit a PR!
### Code of Conduct
- [x] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@seatunnel.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] [Bug] [SqlServer-CDC] Filter transform include_fields not working with generate_sink_sql=true in SqlServer-CDC multi-table scenario [seatunnel]
Posted by "panrobin (via GitHub)" <gi...@apache.org>.
panrobin commented on issue #11774:
URL: https://github.com/apache/seatunnel/issues/11774#issuecomment-5265367161
it works,thx.
SeaTunnel Config
env {
parallelism = 1
job.mode = "STREAMING"
checkpoint.interval = 30000
checkpoint.timeout = 30000
job.retry.times = 3
job.retry.interval.seconds = 3
}
source {
SqlServer-CDC {
plugin_output = "sqlserver_af"
username = "cdcuser"
password = "******"
url = "jdbc:sqlserver://**.**.**.**:1433;databaseName=QG_SC_CIIC_AFOne_Data;encrypt=false;trustServerCertificate=true"
database-names = [
"QG_SC_CIIC_AFOne_Data",
"QG_SC_CIIC_AFONE_MasterData"
]
table-names = [
"QG_SC_CIIC_AFOne_Data.dbo.SYS_User",
"QG_SC_CIIC_AFOne_Data.dbo.SYS_OrgUnit",
"QG_SC_CIIC_AFOne_Data.dbo.SAL_Company",
"QG_SC_CIIC_AFOne_Data.dbo.SAL_ComAdmin",
"QG_SC_CIIC_AFOne_Data.dbo.SAL_Customer",
"QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Employee",
"QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Company",
"QG_SC_CIIC_AFONE_MasterData.dbo.Com_Company"
]
table-names-config = [
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SYS_User", primaryKeys = ["UserId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SYS_OrgUnit", primaryKeys = ["OrgUnitId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SAL_Company", primaryKeys = ["CompanyId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SAL_ComAdmin", primaryKeys = ["ComAdminId"] },
{ table = "QG_SC_CIIC_AFOne_Data.dbo.SAL_Customer", primaryKeys = ["CustomerId"] },
{ table = "QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Employee",primaryKeys = ["ID"] },
{ table = "QG_SC_CIIC_AFONE_MasterData.dbo.Emp_Company", primaryKeys = ["EmpCompanyID"] },
{ table = "QG_SC_CIIC_AFONE_MasterData.dbo.Com_Company", primaryKeys = ["CompanyID"] }
]
startup.mode = "initial"
exactly_once = true
snapshot.split.size = 8096
fetch.size = 1024
poll.interval.ms = 1000
debezium {
database.encrypt = false
database.trustServerCertificate = true
}
}
}
transform {
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "sys_user_only"
database_pattern = "QG_SC_CIIC_AFOne_Data"
schema_pattern = "dbo"
table_pattern = "SYS_User"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "sys_user_only"
plugin_output = "af_sys_user"
include_fields = [
"UserId","Name","EnglishName","LoginName","EmployeeNumber","ADAccount","Password",
"UserType","Email","Mobile","ExtCode","Status","Description","OrgUnitID","OrgUnitCode",
"MainPositionId","ServiceCenterId","ServiceCenterCode","Avatar","PositionType","LoginTime",
"LastLoginTime","ErrorCount","LastErrorTime","IsLock","LockTime","CreatedBy","CreatedTime",
"LastUpdatedBy","LastUpdatedTime","IsDelete","RowVersion","OrgUintAuthority","NCPreparedCode",
"WICPUserId","WICPUserType","TenantId","OperateAuthType","City","LastUserSecrecyUpdateTime",
"LastLoginStatus","LastLockTime","LoginNameByGroup"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "sys_orgunit_only"
database_pattern = "QG_SC_CIIC_AFOne_Data"
schema_pattern = "dbo"
table_pattern = "SYS_OrgUnit"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "sys_orgunit_only"
plugin_output = "af_sys_orgunit"
include_fields = [
"OrgUnitId","OrgUnitCode","OrgUnitName","ShortName","OrgUnitType","ParentOrgUnitID",
"CodePath","Deep","Description","Sort","Disabled","CreatedBy","CreatedTime",
"LastUpdatedBy","LastUpdatedTime","IsDelete","LegalEntityId","RowVersion"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "sal_company_only"
database_pattern = "QG_SC_CIIC_AFOne_Data"
schema_pattern = "dbo"
table_pattern = "SAL_Company"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "sal_company_only"
plugin_output = "af_sal_company"
include_fields = [
"CompanyId","CustomerId","ContractCategoryId","ContractSubCategoryId","LegalEntityId",
"CompanyNo","AfCompanyNo","CustomerName","CompanyNameSuffix","CompanyName","Remark",
"Status","Disabled","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","IsDelete",
"RowVersion","TeamOrgId","MainCustomerId","ComAdminGroupId","ServiceTypeNotes","ShWorkType",
"ShWorkName","ShGjjAccount","ShSbAccount","IstEmploymentOnly","UniformSocialCreditCode",
"EntrustTypeNotes","IsESign","IsSubpackage","SubpackageType","CostCenterID","CostCenterCode",
"CostCenterName","ContractLegalEntityId","DelegateLegalEntityId","StampedEntityCompany",
"ComInvoiceCategoryID","IsTax","TaxRate","IsVATSUTTax","VATSUTTaxRate","EntrustType",
"CompanyType","TemplateId"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "sal_comadmin_only"
database_pattern = "QG_SC_CIIC_AFOne_Data"
schema_pattern = "dbo"
table_pattern = "SAL_ComAdmin"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "sal_comadmin_only"
plugin_output = "af_sal_comadmin"
include_fields = [
"ComAdminId","ComAdminCode","ComAdminName","ComAdminType","Description","OtherName",
"IsFiveHundred","Source","Status","CountryCode","CountryName","FamousProduct","OwnerUserId",
"Remark","Disabled","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","IsDelete",
"RowVersion","IsCertified","EmployeeType","ComAdminGroupName","SiteCode"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "sal_customer_only"
database_pattern = "QG_SC_CIIC_AFOne_Data"
schema_pattern = "dbo"
table_pattern = "SAL_Customer"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "sal_customer_only"
plugin_output = "af_sal_customer"
include_fields = [
"CustomerId","ComAdminId","CustomerNo","CustomerName","IsMain","CustomerStatus","CustomerType",
"UniformSocialCreditCode","BusinessValidDate","LicenseType","LicenseCode","RegisteredCapital",
"LegalPerson","AddressProvence","AddressCity","Address","Telephone","Fax","IndustryCategory",
"Industry","Country","Nature","Industry1","CustomerClass","QualificationType","BpoRemark",
"DepositInfo","Source","SourceVendorName","Remark","EndRemark","EndBy","EndOperateIP",
"Disabled","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","IsDelete","RowVersion",
"ConfirmationDate","IsCentral","CentralCompanyName","CentralManageLevel","firstSignDate",
"IsCompetitor","IsBidding","BiddingProjectNo","VendorOperationMode","VendorOperationModeName",
"MDMCode","MDMMasterID","MDMhmdbz","CNPlatformCode","BindAdminType","IsMainCustomer","MainCustomerID"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "emp_employee_only"
database_pattern = "QG_SC_CIIC_AFONE_MasterData"
schema_pattern = "dbo"
table_pattern = "Emp_Employee"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "emp_employee_only"
plugin_output = "af_emp_employee"
include_fields = [
"ID","EmployeeID","EmployeeName","IDNumber","IDType","Mobile","Birthdate","Gender",
"MaritalStatus","Education","Email","WechatNo","WechatNick","Telephone","ZipCode","Address",
"Province","City","Area","Street","Way","Village","Number","Room","ResidentAddress",
"ResidentArea","ResidentStreet","ResidentWay","ResidentVillage","ResidentNo","ResidentRoom",
"ResidenZipCode","Nationality","Politics","Nation","PhdType","PhdStatus","SeniorEmpNo",
"ChildrenNum","IsResidence","ResidenceType","ResidenceNo","BeforeName","SocialInfo","FundInfo",
"EmpSource","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","RowVersion",
"ResidentProvince","ResidentCity","CertificateStartDate","CertificateEndDate","Major",
"GraduateSchool","GraduateTime","EmployeeTitle","RCGWId","EnglishName","PinYin",
"IsIndustrialInjury","IsVeteran","IsDisability","IsPayroll"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "emp_company_only"
database_pattern = "QG_SC_CIIC_AFONE_MasterData"
schema_pattern = "dbo"
table_pattern = "Emp_Company"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "emp_company_only"
plugin_output = "af_emp_company"
include_fields = [
"EmpCompanyID","EmployeeID","AFCompanyID","CustomerNo","ComAdminCode","FirstDate",
"EmpAttribute","IsTerminated","TerminationDate","TerminationReason","TerminalBy",
"TerminationOperateDate","SocialEndDate","FundEndDate","InsuranceEndDate","Position",
"JobNumber","DeptName","IsApplyCard","Health","Skill","JobProfession","Post","WorkPlace",
"Brand","WorkArea","EmpFullName","SendCardDate","HandelCardDate","Is_NoCard","CostumesMonth",
"CostumesBalance","InDate","ProbationStart","ProbationEnd","Retire55","Rehire","RehireAge",
"Remark","ArTer","ArTerFeedback","ArTerOutDate","SocialStatus","SocialDate","IsSocialHandle",
"SocialNotice","GroupID","DeductGroupID","EmploymentType","EmpSource","CreatedBy","CreatedTime",
"LastUpdatedBy","LastUpdatedTime","RowVersion","EmpNature","ExemptionCreditDate",
"EmployeeIDBefore","EmpRemark","IsViewPermission","IsTransfer","TransferDate","PfileTagID",
"PersonNatrue","EmploymentAttribute","SSContractStartDate","PfileTagStartMonth","PfileTagEndMonth",
"TerminationRemark","TerminationProceduresDate","SourceType","CertificationStatus",
"CertificationFailureReason","DaySalary","HourSalary","IsMigrantWorkers","GraduationDate",
"EmployeeType","PackageID","SSAndHFOutgoingRules","SSAccountType","SSAccountName","City",
"ServiceUserID","ProofSignetID","ProofSignetName","RehireDate","IsLabourServices",
"LabourServicesType","IsApply","LabourMaterials","IsNeedSignContract","ContractType",
"EmpUseType","IsPreResign","ZytEmpId","BusEmploymentType"
]
}
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "com_company_only"
database_pattern = "QG_SC_CIIC_AFONE_MasterData"
schema_pattern = "dbo"
table_pattern = "Com_Company"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "com_company_only"
plugin_output = "af_com_company"
include_fields = [
"CompanyID","AFCompanyID","ComAdminId","ComAdminCode","CustomerId","CustomerNo","CompanyName",
"CompanyEnName","ServiceCenterID","ServiceCentre","OrgUnitID","CusManger","EmpCount",
"ServiceTypeID","ServiceType","CustomerProperty","CompanyNature","IsTerminated","TerminationDate",
"TerminationReason","TerminationOpDate","Remark","EsRemark","CreditPeriod","MgtCreditPeriod",
"PayrollCreditPeriod","IsUpload","IsEHR","CsOperator","IsExpenseSuspend","ReimbursementCard",
"SalaryCard","IsSalary","StatementArea","LegalEntityID","OrgCode","IsIdtEmployment","Counsellor",
"EntryDepart","Industry","TaxDeclareMonth","PayrollDeclareMonth","TaxRulePeriod","CompanyType",
"CompanyAddress","CreatedBy","CreatedTime","LastUpdatedBy","LastUpdatedTime","RowVersion",
"IstEmploymentOnly","CusTeam","SocialSecurityMonth","SocialSecurityDay","SalaryMonth","SalaryDay",
"ServiceFeeMonth","ServiceFeeDay","ServiceTypeNotes","InvoiceWhereabouts","PrincipalPerson",
"IsIntel","ShWorkType","IsContract","SalaryType","ShWorkName","CanInvoice","EntrustTypeNotes",
"BillScope","IsESign","ESignStartDate","ESignEndDate","IsSubpackage","SubpackageType",
"isAutoCreateHealthTask","IsCallCenter","IsSyncESign","LegalSubject","ContractLegalEntityId",
"DelegateLegalEntityId","IsAgreementChangedCreateTask","StampedEntityCompany","IsYDDLKH",
"IsSelfManual","SecTerminationReason","TerTerminationReason","SFSSTYZCYX","ComInvoiceCategoryID",
"AffiliatedCompanyName","SyncESignType"
]
}
}
sink {
Jdbc {
plugin_input = "af_sys_user"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sys_orgunit"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sal_company"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sal_comadmin"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_sal_customer"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_emp_employee"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_emp_company"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
Jdbc {
plugin_input = "af_com_company"
url = "jdbc:mysql://**.**.**.**:3306/ciic_sc?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true"
driver = "com.mysql.cj.jdbc.Driver"
user = "jxpt"
password = "******"
generate_sink_sql = true
database = "ciic_sc"
table = "${table_name}"
support_upsert_by_query_primary_key_exist = true
primary_keys = ["${primary_key}"]
batch_size = 500
batch_interval_ms = 1000
max_retries = 3
is_exactly_once = false
}
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@seatunnel.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] [Bug] [SqlServer-CDC] Filter transform include_fields not working with generate_sink_sql=true in SqlServer-CDC multi-table scenario [seatunnel]
Posted by "davidzollo (via GitHub)" <gi...@apache.org>.
davidzollo closed issue #11774: [Bug] [SqlServer-CDC] Filter transform include_fields not working with generate_sink_sql=true in SqlServer-CDC multi-table scenario
URL: https://github.com/apache/seatunnel/issues/11774
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@seatunnel.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] [Bug] [SqlServer-CDC] Filter transform include_fields not working with generate_sink_sql=true in SqlServer-CDC multi-table scenario [seatunnel]
Posted by "nzw921rx (via GitHub)" <gi...@apache.org>.
nzw921rx commented on issue #11774:
URL: https://github.com/apache/seatunnel/issues/11774#issuecomment-5264788506
Thanks for the detailed report. I checked the configuration and the SeaTunnel 2.3.13 implementation. The current behavior is caused by the multi-table transform routing semantics rather than `table_match_regex` or `generate_sink_sql`.
The following regular expression is valid:
```hocon
table_match_regex = ".*QG_SC_CIIC_AFOne_Data\\.dbo\\.SAL_Company"
```
After HOCON unescaping, it becomes:
```text
.*QG_SC_CIIC_AFOne_Data\.dbo\.SAL_Company
```
This matches the actual table path:
```text
QG_SC_CIIC_AFOne_Data.dbo.SAL_Company
```
The leading `.*` is unnecessary, but it does not prevent the match.
The important behavior is in [`[AbstractMultiCatalogTransform](https://github.com/apache/seatunnel/blob/2.3.13/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractMultiCatalogTransform.java)`](https://github.com/apache/seatunnel/blob/2.3.13/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/common/AbstractMultiCatalogTransform.java): `table_match_regex` only determines whether the transform is applied to a table. Tables that do not match are handled by `IdentityMapTransform` and continue downstream unchanged; they are not removed from the transform output.
All eight `Filter` transforms use the same input:
```hocon
plugin_input = "sqlserver_af"
```
Therefore, each transform output still contains all eight source tables:
* In the `af_sal_company` branch, `SAL_Company` matches the regular expression and `BillMaker` is removed.
* In the other seven branches, `SAL_Company` does not match their regular expressions, so it is passed through unchanged and still contains `BillMaker`.
* Each JDBC sink uses `table = "${table_name}"`, so those sinks may also attempt to write the unchanged `SAL_Company` table.
The generated SQL containing the configured fields plus `BillMaker` is consistent with the schema coming from one of these pass-through branches.
To create a separate branch for each table, use `TableFilter` to retain only the intended table before applying the field `Filter`. For example:
```hocon
transform {
TableFilter {
plugin_input = "sqlserver_af"
plugin_output = "sal_company_only"
database_pattern = "QG_SC_CIIC_AFOne_Data"
schema_pattern = "dbo"
table_pattern = "SAL_Company"
pattern_mode = "INCLUDE"
}
Filter {
plugin_input = "sal_company_only"
plugin_output = "af_sal_company"
include_fields = [
"CompanyId",
"CustomerId",
"TemplateId"
# other required fields
]
}
}
```
The existing JDBC sink can continue using:
```hocon
plugin_input = "af_sal_company"
```
Please apply the same `TableFilter -> Filter -> JDBC Sink` structure to the other table branches. No changes are required in `table_match_regex`, the `Filter` implementation, or `generate_sink_sql`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@seatunnel.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org