You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by lz...@apache.org on 2022/08/10 10:45:29 UTC
[inlong] branch master updated: [INLONG-5456][Dashboard] Support source/sink metadata management (#5457)
This is an automated email from the ASF dual-hosted git repository.
lzwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 7bc1674bf [INLONG-5456][Dashboard] Support source/sink metadata management (#5457)
7bc1674bf is described below
commit 7bc1674bff082a865a2d917ae2211421c4efb904
Author: Daniel <le...@apache.org>
AuthorDate: Wed Aug 10 18:45:24 2022 +0800
[INLONG-5456][Dashboard] Support source/sink metadata management (#5457)
* [INLONG-5456][Dashboard] Support source/sink metadata management
* fix: ts type error.
---
.../AccessHelper/DataSourcesEditor/CreateModal.tsx | 63 +++++++++++-----------
.../AccessHelper/DataSourcesEditor/index.tsx | 19 +++++--
.../AccessHelper/DataStorageEditor/DetailModal.tsx | 4 +-
.../AccessHelper/DataStorageEditor/Editor.tsx | 2 +-
.../AccessHelper/FieldsConfig/dataFields.tsx | 2 +-
.../sinks/clickhouse.tsx} | 6 +--
.../sinks/common/sourceFields.ts} | 2 +-
.../MetaData/StorageEs.tsx => meta/sinks/es.tsx} | 6 +--
.../sinks/greenplum.tsx} | 6 +--
.../StorageHBase.tsx => meta/sinks/hbase.tsx} | 6 +--
.../StorageHive.tsx => meta/sinks/hive.tsx} | 6 +--
.../StorageIceberg.tsx => meta/sinks/iceberg.tsx} | 6 +--
.../{components/MetaData => meta/sinks}/index.ts | 48 ++++++++---------
.../StorageKafka.tsx => meta/sinks/kafka.tsx} | 2 +-
.../StorageMySQL.tsx => meta/sinks/mysql.tsx} | 6 +--
.../StorageOracle.tsx => meta/sinks/oracle.tsx} | 6 +--
.../sinks/postgreSql.tsx} | 6 +--
.../sinks/sqlServer.tsx} | 6 +--
.../sinks/tdsqlPostgreSql.tsx} | 6 +--
inlong-dashboard/src/meta/sources/autoPush.ts | 34 ++++++++++++
.../sources/binLog.ts} | 18 ++++---
.../DataSourcesFile.ts => meta/sources/file.ts} | 12 +++--
inlong-dashboard/src/meta/sources/index.ts | 55 +++++++++++++++++++
.../src/pages/AccessDetail/DataSources/index.tsx | 41 ++++++++------
.../src/pages/AccessDetail/DataStorage/index.tsx | 2 +-
25 files changed, 241 insertions(+), 129 deletions(-)
diff --git a/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/CreateModal.tsx b/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/CreateModal.tsx
index feb0629ad..a577adec3 100644
--- a/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/CreateModal.tsx
+++ b/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/CreateModal.tsx
@@ -23,13 +23,8 @@ import { ModalProps } from 'antd/es/modal';
import FormGenerator, { useForm } from '@/components/FormGenerator';
import { useRequest, useUpdateEffect } from '@/hooks';
import { useTranslation } from 'react-i18next';
-import { getDataSourcesFileFields as getFileCreateFormContent } from '@/components/MetaData/DataSourcesFile';
-import {
- getDataSourcesBinLogFields,
- toFormValues,
- toSubmitValues,
-} from '@/components/MetaData/DataSourcesBinLog';
import { FormItemProps } from '@/components/FormGenerator';
+import { sources, SourceType } from '@/meta/sources';
export interface Props extends ModalProps {
type: 'MYSQL_BINLOG' | 'FILE';
@@ -41,6 +36,14 @@ export interface Props extends ModalProps {
content?: FormItemProps[];
}
+const sourcesMap: Record<string, SourceType> = sources.reduce(
+ (acc, cur) => ({
+ ...acc,
+ [cur.value]: cur,
+ }),
+ {},
+);
+
const Comp: React.FC<Props> = ({ type, id, content = [], record, ...modalProps }) => {
const [form] = useForm();
const { t } = useTranslation();
@@ -49,9 +52,7 @@ const Comp: React.FC<Props> = ({ type, id, content = [], record, ...modalProps }
const toFormVals = useCallback(
v => {
- const mapFunc = {
- MYSQL_BINLOG: toFormValues,
- }[type];
+ const mapFunc = sourcesMap[type]?.toFormValues;
return mapFunc ? mapFunc(v) : v;
},
[type],
@@ -59,9 +60,7 @@ const Comp: React.FC<Props> = ({ type, id, content = [], record, ...modalProps }
const toSubmitVals = useCallback(
v => {
- const mapFunc = {
- MYSQL_BINLOG: toSubmitValues,
- }[type];
+ const mapFunc = sourcesMap[type]?.toSubmitValues;
return mapFunc ? mapFunc(v) : v;
},
[type],
@@ -105,26 +104,24 @@ const Comp: React.FC<Props> = ({ type, id, content = [], record, ...modalProps }
}
}, [modalProps.visible]);
- const getCreateFormContent = useMemo(
- () => currentValues => {
- const config = {
- MYSQL_BINLOG: getDataSourcesBinLogFields,
- FILE: getFileCreateFormContent,
- }[type]('form', { currentValues }) as FormItemProps[];
- return [
- {
- name: 'sourceName',
- type: 'input',
- label: t('components.AccessHelper.DataSourcesEditor.CreateModal.DataSourceName'),
- rules: [{ required: true }],
- props: {
- disabled: !!id,
- },
- } as FormItemProps,
- ].concat(config);
- },
- [type, id, t],
- );
+ const formContent = useMemo(() => {
+ const getForm = sourcesMap[type].getForm;
+ const config = getForm('form', {
+ currentValues,
+ form,
+ }) as FormItemProps[];
+ return [
+ {
+ name: 'sourceName',
+ type: 'input',
+ label: t('components.AccessHelper.DataSourcesEditor.CreateModal.DataSourceName'),
+ rules: [{ required: true }],
+ props: {
+ disabled: !!id,
+ },
+ } as FormItemProps,
+ ].concat(config);
+ }, [type, id, currentValues, form, t]);
return (
<>
@@ -139,7 +136,7 @@ const Comp: React.FC<Props> = ({ type, id, content = [], record, ...modalProps }
onOk={onOk}
>
<FormGenerator
- content={content.concat(getCreateFormContent(currentValues))}
+ content={content.concat(formContent)}
onValuesChange={vals => setCurrentValues(prev => ({ ...prev, ...vals }))}
allValues={currentValues}
form={form}
diff --git a/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/index.tsx b/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/index.tsx
index 4940a25e7..f4d3c85c1 100644
--- a/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/index.tsx
+++ b/inlong-dashboard/src/components/AccessHelper/DataSourcesEditor/index.tsx
@@ -17,13 +17,12 @@
* under the License.
*/
-import React, { useState } from 'react';
+import React, { useState, useMemo } from 'react';
import { Button, Table, Modal, message } from 'antd';
import { useTranslation } from 'react-i18next';
import request from '@/utils/request';
import { useUpdateEffect, usePrevious } from '@/hooks';
-import { dataSourcesBinLogColumns as binLogColumns } from '@/components/MetaData/DataSourcesBinLog';
-import { dataSourcesFileColumns as fileColumns } from '@/components/MetaData/DataSourcesFile';
+import { sources } from '@/meta/sources';
import CreateModal from './CreateModal';
export interface DataSourcesEditorProps {
@@ -162,7 +161,19 @@ const Comp = ({
setData(newData);
};
- const columns = (type === 'MYSQL_BINLOG' ? binLogColumns : fileColumns).concat(
+ const columnsMap = useMemo(
+ () =>
+ sources.reduce(
+ (acc, cur) => ({
+ ...acc,
+ [cur.value]: cur.tableColumns,
+ }),
+ {},
+ ),
+ [],
+ );
+
+ const columns = columnsMap[type].concat(
readonly
? []
: [
diff --git a/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx b/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx
index 69a42e4ba..eb624366b 100644
--- a/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx
+++ b/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx
@@ -27,7 +27,7 @@ import FormGenerator, {
FormItemProps,
FormGeneratorProps,
} from '@/components/FormGenerator';
-import { Storages, StoragesType } from '@/components/MetaData';
+import { Storages, StoragesType } from '@/meta/sinks';
export interface DetailModalProps extends ModalProps {
inlongGroupId: string;
@@ -37,7 +37,7 @@ export interface DetailModalProps extends ModalProps {
id?: string;
// (False operation) Need to pass when editing, row data
record?: Record<string, any>;
- sinkType: 'HIVE' | 'TEST';
+ sinkType: string;
dataType?: string;
// defaultRowTypeFields, which can be used to auto-fill form default values
defaultRowTypeFields?: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/Editor.tsx b/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/Editor.tsx
index 6339f76d0..e722cfcb1 100644
--- a/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/Editor.tsx
+++ b/inlong-dashboard/src/components/AccessHelper/DataStorageEditor/Editor.tsx
@@ -23,7 +23,7 @@ import request from '@/utils/request';
import isEqual from 'lodash/isEqual';
import { useTranslation } from 'react-i18next';
import DetailModal from './DetailModal';
-import { Storages } from '@/components/MetaData';
+import { Storages } from '@/meta/sinks';
export interface Props {
value?: Record<string, any>[];
diff --git a/inlong-dashboard/src/components/AccessHelper/FieldsConfig/dataFields.tsx b/inlong-dashboard/src/components/AccessHelper/FieldsConfig/dataFields.tsx
index 77f375b06..6282af3db 100644
--- a/inlong-dashboard/src/components/AccessHelper/FieldsConfig/dataFields.tsx
+++ b/inlong-dashboard/src/components/AccessHelper/FieldsConfig/dataFields.tsx
@@ -22,7 +22,7 @@ import { FormItemProps } from '@/components/FormGenerator';
import { pickObjectArray } from '@/utils';
import EditableTable from '@/components/EditableTable';
import i18n from '@/i18n';
-import { fieldTypes as sourceFieldsTypes } from '@/components/MetaData/SourceDataFields';
+import { fieldTypes as sourceFieldsTypes } from '@/meta/sinks/common/sourceFields';
type RestParams = {
// Whether the fieldList is in edit mode
diff --git a/inlong-dashboard/src/components/MetaData/StorageClickhouse.tsx b/inlong-dashboard/src/meta/sinks/clickhouse.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageClickhouse.tsx
rename to inlong-dashboard/src/meta/sinks/clickhouse.tsx
index da3c8914d..f71df3855 100644
--- a/inlong-dashboard/src/components/MetaData/StorageClickhouse.tsx
+++ b/inlong-dashboard/src/meta/sinks/clickhouse.tsx
@@ -26,7 +26,7 @@ import i18n from '@/i18n';
import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// ClickHouse targetType
const clickhouseTargetTypes = [
@@ -266,7 +266,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `ClickHouse${i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.FieldName')}`,
dataIndex: 'fieldName',
@@ -342,7 +342,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageClickhouse = {
+export const clickhouse = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/SourceDataFields.ts b/inlong-dashboard/src/meta/sinks/common/sourceFields.ts
similarity index 97%
rename from inlong-dashboard/src/components/MetaData/SourceDataFields.ts
rename to inlong-dashboard/src/meta/sinks/common/sourceFields.ts
index 46b793b92..f72bba8b0 100644
--- a/inlong-dashboard/src/components/MetaData/SourceDataFields.ts
+++ b/inlong-dashboard/src/meta/sinks/common/sourceFields.ts
@@ -27,7 +27,7 @@ export const fieldTypes = ['int', 'long', 'float', 'double', 'string', 'date', '
}),
);
-export const sourceDataFields: ColumnsItemProps[] = [
+export const sourceFields: ColumnsItemProps[] = [
{
title: i18n.t('components.AccessHelper.StorageMetaData.SourceFieldName'),
dataIndex: 'sourceFieldName',
diff --git a/inlong-dashboard/src/components/MetaData/StorageEs.tsx b/inlong-dashboard/src/meta/sinks/es.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageEs.tsx
rename to inlong-dashboard/src/meta/sinks/es.tsx
index ae3d49b1c..e2c90b133 100644
--- a/inlong-dashboard/src/components/MetaData/StorageEs.tsx
+++ b/inlong-dashboard/src/meta/sinks/es.tsx
@@ -26,7 +26,7 @@ import i18n from '@/i18n';
import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
const esTypes = [
'text',
@@ -179,7 +179,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `ES ${i18n.t('components.AccessHelper.StorageMetaData.Es.FieldName')}`,
dataIndex: 'fieldName',
@@ -250,7 +250,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageEs = {
+export const es = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageGreenplum.tsx b/inlong-dashboard/src/meta/sinks/greenplum.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageGreenplum.tsx
rename to inlong-dashboard/src/meta/sinks/greenplum.tsx
index c476fdeff..bddc6ff79 100644
--- a/inlong-dashboard/src/components/MetaData/StorageGreenplum.tsx
+++ b/inlong-dashboard/src/meta/sinks/greenplum.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// greenplumFieldTypes
const greenplumFieldTypes = [
@@ -156,7 +156,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `GREENPLUM${i18n.t('components.AccessHelper.StorageMetaData.Greenplum.FieldName')}`,
dataIndex: 'fieldName',
@@ -225,7 +225,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageGreenplum = {
+export const greenplum = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageHBase.tsx b/inlong-dashboard/src/meta/sinks/hbase.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageHBase.tsx
rename to inlong-dashboard/src/meta/sinks/hbase.tsx
index 81b9a01d2..04d09a7a1 100644
--- a/inlong-dashboard/src/components/MetaData/StorageHBase.tsx
+++ b/inlong-dashboard/src/meta/sinks/hbase.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// hbaseFieldTypes
const hbaseFieldTypes = [
@@ -157,7 +157,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `HBASE${i18n.t('components.AccessHelper.StorageMetaData.HBase.FieldName')}`,
dataIndex: 'fieldName',
@@ -208,7 +208,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageHBase = {
+export const hbase = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageHive.tsx b/inlong-dashboard/src/meta/sinks/hive.tsx
similarity index 99%
rename from inlong-dashboard/src/components/MetaData/StorageHive.tsx
rename to inlong-dashboard/src/meta/sinks/hive.tsx
index fca2a87a9..3f3e827f2 100644
--- a/inlong-dashboard/src/components/MetaData/StorageHive.tsx
+++ b/inlong-dashboard/src/meta/sinks/hive.tsx
@@ -29,7 +29,7 @@ import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
// import request from '@/utils/request';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// hiveFieldTypes
const hiveFieldTypes = [
@@ -344,7 +344,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `HIVE${i18n.t('components.AccessHelper.StorageMetaData.Hive.FieldName')}`,
dataIndex: 'fieldName',
@@ -413,7 +413,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageHive = {
+export const hive = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageIceberg.tsx b/inlong-dashboard/src/meta/sinks/iceberg.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageIceberg.tsx
rename to inlong-dashboard/src/meta/sinks/iceberg.tsx
index 259ca8d5e..4694bfb31 100644
--- a/inlong-dashboard/src/components/MetaData/StorageIceberg.tsx
+++ b/inlong-dashboard/src/meta/sinks/iceberg.tsx
@@ -28,7 +28,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable from '@/components/EditableTable';
import { excludeObject } from '@/utils';
import TextSwitch from '@/components/TextSwitch';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// import { Button, message } from 'antd';
// import request from '@/utils/request';
@@ -285,7 +285,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `Iceberg ${i18n.t('components.AccessHelper.StorageMetaData.Iceberg.FieldName')}`,
width: 110,
@@ -373,7 +373,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageIceberg = {
+export const iceberg = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/index.ts b/inlong-dashboard/src/meta/sinks/index.ts
similarity index 71%
rename from inlong-dashboard/src/components/MetaData/index.ts
rename to inlong-dashboard/src/meta/sinks/index.ts
index 0fc5cc8a0..7bfc886a6 100644
--- a/inlong-dashboard/src/components/MetaData/index.ts
+++ b/inlong-dashboard/src/meta/sinks/index.ts
@@ -19,18 +19,18 @@
import type { GetStorageFormFieldsType, GetStorageColumnsType } from '@/utils/metaData';
import type { ColumnsType } from 'antd/es/table';
-import { StorageHive } from './StorageHive';
-import { StorageClickhouse } from './StorageClickhouse';
-import { StorageKafka } from './StorageKafka';
-import { StorageIceberg } from './StorageIceberg';
-import { StorageEs } from './StorageEs';
-import { StorageGreenplum } from './StorageGreenplum';
-import { StorageMySQL } from './StorageMySQL';
-import { StorageOracle } from './StorageOracle';
-import { StoragePostgreSQL } from './StoragePostgreSQL';
-import { StorageSQLServer } from './StorageSQLServer';
-import { StorageTDSQLPostgreSQL } from './StorageTDSQLPostgreSQL';
-import { StorageHBase } from './StorageHBase';
+import { hive } from './hive';
+import { clickhouse } from './clickhouse';
+import { kafka } from './kafka';
+import { iceberg } from './iceberg';
+import { es } from './es';
+import { greenplum } from './greenplum';
+import { mysql } from './mysql';
+import { oracle } from './oracle';
+import { postgreSql } from './postgreSql';
+import { sqlServer } from './sqlServer';
+import { tdsqlPostgreSQL } from './tdsqlPostgreSql';
+import { hbase } from './hbase';
export interface StoragesType {
label: string;
@@ -51,61 +51,61 @@ export const Storages: StoragesType[] = [
{
label: 'Hive',
value: 'HIVE',
- ...StorageHive,
+ ...hive,
},
{
label: 'Iceberg',
value: 'ICEBERG',
- ...StorageIceberg,
+ ...iceberg,
},
{
label: 'ClickHouse',
value: 'CLICKHOUSE',
- ...StorageClickhouse,
+ ...clickhouse,
},
{
label: 'Kafka',
value: 'KAFKA',
- ...StorageKafka,
+ ...kafka,
},
{
label: 'Elasticsearch',
value: 'ELASTICSEARCH',
- ...StorageEs,
+ ...es,
},
{
label: 'Greenplum',
value: 'GREENPLUM',
- ...StorageGreenplum,
+ ...greenplum,
},
{
label: 'HBase',
value: 'HBASE',
- ...StorageHBase,
+ ...hbase,
},
{
label: 'MySQL',
value: 'MYSQL',
- ...StorageMySQL,
+ ...mysql,
},
{
label: 'Oracle',
value: 'ORACLE',
- ...StorageOracle,
+ ...oracle,
},
{
label: 'PostgreSQL',
value: 'POSTGRES',
- ...StoragePostgreSQL,
+ ...postgreSql,
},
{
label: 'SQLServer',
value: 'SQLSERVER',
- ...StorageSQLServer,
+ ...sqlServer,
},
{
label: 'TDSQLPostgreSQL',
value: 'TDSQLPOSTGRESQL',
- ...StorageTDSQLPostgreSQL,
+ ...tdsqlPostgreSQL,
},
];
diff --git a/inlong-dashboard/src/components/MetaData/StorageKafka.tsx b/inlong-dashboard/src/meta/sinks/kafka.tsx
similarity index 99%
rename from inlong-dashboard/src/components/MetaData/StorageKafka.tsx
rename to inlong-dashboard/src/meta/sinks/kafka.tsx
index 277a150e7..dc0b0e0f8 100644
--- a/inlong-dashboard/src/components/MetaData/StorageKafka.tsx
+++ b/inlong-dashboard/src/meta/sinks/kafka.tsx
@@ -118,7 +118,7 @@ const getForm: GetStorageFormFieldsType = (
const tableColumns = getForm('col') as ColumnsType;
-export const StorageKafka = {
+export const kafka = {
getForm,
tableColumns,
};
diff --git a/inlong-dashboard/src/components/MetaData/StorageMySQL.tsx b/inlong-dashboard/src/meta/sinks/mysql.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageMySQL.tsx
rename to inlong-dashboard/src/meta/sinks/mysql.tsx
index c71b48312..555e7420d 100644
--- a/inlong-dashboard/src/components/MetaData/StorageMySQL.tsx
+++ b/inlong-dashboard/src/meta/sinks/mysql.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// mysqlFieldTypes
const mysqlFieldTypes = [
@@ -152,7 +152,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `MYSQL${i18n.t('components.AccessHelper.StorageMetaData.MySQL.FieldName')}`,
dataIndex: 'fieldName',
@@ -221,7 +221,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageMySQL = {
+export const mysql = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageOracle.tsx b/inlong-dashboard/src/meta/sinks/oracle.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageOracle.tsx
rename to inlong-dashboard/src/meta/sinks/oracle.tsx
index b502c3506..727cf289a 100644
--- a/inlong-dashboard/src/components/MetaData/StorageOracle.tsx
+++ b/inlong-dashboard/src/meta/sinks/oracle.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// oracleFieldTypes
const oracleFieldTypes = [
@@ -151,7 +151,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `ORACLE${i18n.t('components.AccessHelper.StorageMetaData.Oracle.FieldName')}`,
dataIndex: 'fieldName',
@@ -220,7 +220,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageOracle = {
+export const oracle = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StoragePostgreSQL.tsx b/inlong-dashboard/src/meta/sinks/postgreSql.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StoragePostgreSQL.tsx
rename to inlong-dashboard/src/meta/sinks/postgreSql.tsx
index d71addd79..079bb5e5c 100644
--- a/inlong-dashboard/src/components/MetaData/StoragePostgreSQL.tsx
+++ b/inlong-dashboard/src/meta/sinks/postgreSql.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// postgreSQLFieldTypes
const postgreSQLFieldTypes = [
@@ -165,7 +165,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `POSTGRESQL${i18n.t('components.AccessHelper.StorageMetaData.PostgreSQL.FieldName')}`,
dataIndex: 'fieldName',
@@ -234,7 +234,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StoragePostgreSQL = {
+export const postgreSql = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageSQLServer.tsx b/inlong-dashboard/src/meta/sinks/sqlServer.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageSQLServer.tsx
rename to inlong-dashboard/src/meta/sinks/sqlServer.tsx
index e2472f22e..1b367daf6 100644
--- a/inlong-dashboard/src/components/MetaData/StorageSQLServer.tsx
+++ b/inlong-dashboard/src/meta/sinks/sqlServer.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// sqlserverFieldTypes
const sqlserverFieldTypes = [
@@ -198,7 +198,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `SQLSERVER${i18n.t('components.AccessHelper.StorageMetaData.SQLServer.FieldName')}`,
dataIndex: 'fieldName',
@@ -267,7 +267,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageSQLServer = {
+export const sqlServer = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/components/MetaData/StorageTDSQLPostgreSQL.tsx b/inlong-dashboard/src/meta/sinks/tdsqlPostgreSql.tsx
similarity index 98%
rename from inlong-dashboard/src/components/MetaData/StorageTDSQLPostgreSQL.tsx
rename to inlong-dashboard/src/meta/sinks/tdsqlPostgreSql.tsx
index c49b32d4d..eb98b87b7 100644
--- a/inlong-dashboard/src/components/MetaData/StorageTDSQLPostgreSQL.tsx
+++ b/inlong-dashboard/src/meta/sinks/tdsqlPostgreSql.tsx
@@ -25,7 +25,7 @@ import { ColumnsType } from 'antd/es/table';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import i18n from '@/i18n';
import { excludeObject } from '@/utils';
-import { sourceDataFields } from './SourceDataFields';
+import { sourceFields } from './common/sourceFields';
// tdsqlpostgreSQLFieldTypes
const tdsqlpostgreSQLFieldTypes = [
@@ -165,7 +165,7 @@ const getForm: GetStorageFormFieldsType = (
const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) => {
return [
- ...sourceDataFields,
+ ...sourceFields,
{
title: `TDSQLPOSTGRESQL${i18n.t(
'components.AccessHelper.StorageMetaData.TDSQLPostgreSQL.FieldName',
@@ -238,7 +238,7 @@ const getFieldListColumns: GetStorageColumnsType = (dataType, currentValues) =>
const tableColumns = getForm('col') as ColumnsType;
-export const StorageTDSQLPostgreSQL = {
+export const tdsqlPostgreSQL = {
getForm,
getFieldListColumns,
tableColumns,
diff --git a/inlong-dashboard/src/meta/sources/autoPush.ts b/inlong-dashboard/src/meta/sources/autoPush.ts
new file mode 100644
index 000000000..c2580da12
--- /dev/null
+++ b/inlong-dashboard/src/meta/sources/autoPush.ts
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getColsFromFields } from '@/utils/metaData';
+import { ColumnsType } from 'antd/es/table';
+
+const getForm = (type: 'form' | 'col' = 'form', { currentValues } = {} as any) => {
+ const fileds = [];
+
+ return type === 'col' ? getColsFromFields(fileds) : fileds;
+};
+
+const tableColumns = getForm('col') as ColumnsType;
+
+export const autoPush = {
+ getForm,
+ tableColumns,
+};
diff --git a/inlong-dashboard/src/components/MetaData/DataSourcesBinLog.ts b/inlong-dashboard/src/meta/sources/binLog.ts
similarity index 94%
rename from inlong-dashboard/src/components/MetaData/DataSourcesBinLog.ts
rename to inlong-dashboard/src/meta/sources/binLog.ts
index de01ca276..a436d4ffb 100644
--- a/inlong-dashboard/src/components/MetaData/DataSourcesBinLog.ts
+++ b/inlong-dashboard/src/meta/sources/binLog.ts
@@ -22,10 +22,7 @@ import { getColsFromFields } from '@/utils/metaData';
import { ColumnsType } from 'antd/es/table';
import i18n from '@/i18n';
-export const getDataSourcesBinLogFields = (
- type: 'form' | 'col' = 'form',
- { currentValues } = {} as any,
-) => {
+const getForm = (type: 'form' | 'col' = 'form', { currentValues } = {} as any) => {
const fileds = [
{
name: 'hostname',
@@ -150,14 +147,14 @@ export const getDataSourcesBinLogFields = (
return type === 'col' ? getColsFromFields(fileds) : fileds;
};
-export const toFormValues = data => {
+const toFormValues = data => {
return {
...data,
_startDumpPosition: data.startDumpPosition ? 1 : 0,
};
};
-export const toSubmitValues = data => {
+const toSubmitValues = data => {
const output = { ...data };
delete output._startDumpPosition;
return {
@@ -166,4 +163,11 @@ export const toSubmitValues = data => {
};
};
-export const dataSourcesBinLogColumns = getDataSourcesBinLogFields('col') as ColumnsType;
+const tableColumns = getForm('col') as ColumnsType;
+
+export const binLog = {
+ getForm,
+ tableColumns,
+ toFormValues,
+ toSubmitValues,
+};
diff --git a/inlong-dashboard/src/components/MetaData/DataSourcesFile.ts b/inlong-dashboard/src/meta/sources/file.ts
similarity index 90%
rename from inlong-dashboard/src/components/MetaData/DataSourcesFile.ts
rename to inlong-dashboard/src/meta/sources/file.ts
index 86ec27d01..d9aa6991d 100644
--- a/inlong-dashboard/src/components/MetaData/DataSourcesFile.ts
+++ b/inlong-dashboard/src/meta/sources/file.ts
@@ -22,10 +22,7 @@ import { ColumnsType } from 'antd/es/table';
import rulesPattern from '@/utils/pattern';
import i18n from '@/i18n';
-export const getDataSourcesFileFields = (
- type: 'form' | 'col' = 'form',
- { currentValues } = {} as any,
-) => {
+const getForm = (type: 'form' | 'col' = 'form', { currentValues } = {} as any) => {
const fileds = [
{
type: 'input',
@@ -60,4 +57,9 @@ export const getDataSourcesFileFields = (
return type === 'col' ? getColsFromFields(fileds) : fileds;
};
-export const dataSourcesFileColumns = getDataSourcesFileFields('col') as ColumnsType;
+const tableColumns = getForm('col') as ColumnsType;
+
+export const file = {
+ getForm,
+ tableColumns,
+};
diff --git a/inlong-dashboard/src/meta/sources/index.ts b/inlong-dashboard/src/meta/sources/index.ts
new file mode 100644
index 000000000..6ad92715f
--- /dev/null
+++ b/inlong-dashboard/src/meta/sources/index.ts
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import type { GetStorageFormFieldsType } from '@/utils/metaData';
+import type { ColumnsType } from 'antd/es/table';
+import { autoPush } from './autoPush';
+import { binLog } from './binLog';
+import { file } from './file';
+
+export interface SourceType {
+ label: string;
+ value: string;
+ // Generate form configuration for single data
+ getForm: GetStorageFormFieldsType;
+ // Generate table display configuration
+ tableColumns: ColumnsType;
+ // Custom convert interface data to front-end data format
+ toFormValues?: (values: unknown) => unknown;
+ // Custom convert front-end data to interface data format
+ toSubmitValues?: (values: unknown) => unknown;
+}
+
+export const sources: SourceType[] = [
+ {
+ label: 'MySQL BinLog',
+ value: 'MYSQL_BINLOG',
+ ...binLog,
+ },
+ {
+ label: 'File',
+ value: 'FILE',
+ ...file,
+ },
+ {
+ label: 'Auto Push',
+ value: 'AUTO_PUSH',
+ ...autoPush,
+ },
+];
diff --git a/inlong-dashboard/src/pages/AccessDetail/DataSources/index.tsx b/inlong-dashboard/src/pages/AccessDetail/DataSources/index.tsx
index bef3b3517..362e31ef2 100644
--- a/inlong-dashboard/src/pages/AccessDetail/DataSources/index.tsx
+++ b/inlong-dashboard/src/pages/AccessDetail/DataSources/index.tsx
@@ -17,14 +17,13 @@
* under the License.
*/
-import React, { useState, forwardRef } from 'react';
+import React, { useState, forwardRef, useMemo } from 'react';
import { Button, Modal, message } from 'antd';
import HighTable from '@/components/HighTable';
import { defaultSize } from '@/configs/pagination';
import { useRequest } from '@/hooks';
import { DataSourcesCreateModal } from '@/components/AccessHelper';
-import { dataSourcesBinLogColumns } from '@/components/MetaData/DataSourcesBinLog';
-import { dataSourcesFileColumns } from '@/components/MetaData/DataSourcesFile';
+import { sources } from '@/meta/sources';
import i18n from '@/i18n';
import request from '@/utils/request';
import { CommonInterface } from '../common';
@@ -44,16 +43,10 @@ const getFilterFormContent = defaultValues => [
initialValue: defaultValues.sourceType,
props: {
buttonStyle: 'solid',
- options: [
- {
- label: 'File',
- value: 'FILE',
- },
- {
- label: 'MySQL BinLog',
- value: 'MYSQL_BINLOG',
- },
- ],
+ options: sources.map(item => ({
+ label: item.label,
+ value: item.value,
+ })),
},
},
{
@@ -73,7 +66,7 @@ const Comp = ({ inlongGroupId, readonly }: Props, ref) => {
// keyword: '',
pageSize: defaultSize,
pageNum: 1,
- sourceType: 'FILE',
+ sourceType: sources[0].value,
});
const [createModal, setCreateModal] = useState<Record<string, unknown>>({
@@ -156,13 +149,29 @@ const Comp = ({ inlongGroupId, readonly }: Props, ref) => {
total: data?.total,
};
+ const columnsMap = useMemo(
+ () =>
+ sources.reduce(
+ (acc, cur) => ({
+ ...acc,
+ [cur.value]: cur.tableColumns,
+ }),
+ {},
+ ),
+ [],
+ );
+
const columns = [
{
title: i18n.t('pages.AccessDetail.DataSources.DataStreams'),
dataIndex: 'inlongStreamId',
- } as any,
+ },
+ {
+ title: i18n.t('components.AccessHelper.DataSourcesEditor.CreateModal.DataSourceName'),
+ dataIndex: 'sourceName',
+ },
]
- .concat(options.sourceType === 'FILE' ? dataSourcesFileColumns : dataSourcesBinLogColumns)
+ .concat(columnsMap[options.sourceType])
.concat([
{
title: i18n.t('basic.Status'),
diff --git a/inlong-dashboard/src/pages/AccessDetail/DataStorage/index.tsx b/inlong-dashboard/src/pages/AccessDetail/DataStorage/index.tsx
index 1e6877a0b..01b13f3ce 100644
--- a/inlong-dashboard/src/pages/AccessDetail/DataStorage/index.tsx
+++ b/inlong-dashboard/src/pages/AccessDetail/DataStorage/index.tsx
@@ -24,7 +24,7 @@ import { defaultSize } from '@/configs/pagination';
import { useRequest } from '@/hooks';
import i18n from '@/i18n';
import { DataStorageDetailModal } from '@/components/AccessHelper';
-import { Storages } from '@/components/MetaData';
+import { Storages } from '@/meta/sinks';
import request from '@/utils/request';
import { CommonInterface } from '../common';
import { statusList, genStatusTag } from './status';