You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by go...@apache.org on 2021/11/16 07:19:45 UTC
[incubator-inlong] branch master updated: [INLONG-1518] WebSite Support ClickHouse (#1800)
This is an automated email from the ASF dual-hosted git repository.
gosonzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 54086f8 [INLONG-1518] WebSite Support ClickHouse (#1800)
54086f8 is described below
commit 54086f84fb67a09a326937bf147e26bfd95a5f50
Author: Daniel <le...@outlook.com>
AuthorDate: Tue Nov 16 15:19:41 2021 +0800
[INLONG-1518] WebSite Support ClickHouse (#1800)
---
.../AccessHelper/DataStorageEditor/DetailModal.tsx | 21 +-
.../AccessHelper/FieldsConfig/dataFields.tsx | 6 +-
.../src/components/MetaData/DataSourcesFile.ts | 16 +-
.../src/components/MetaData/SourceDataFields.ts | 9 +-
.../src/components/MetaData/StorageClickhouse.tsx | 280 +++++++++++++++++++++
.../src/components/MetaData/StorageHive.tsx | 246 +++++++++---------
inlong-website/src/locales/cn.json | 63 +++--
inlong-website/src/locales/en.json | 63 +++--
.../src/pages/AccessDetail/DataStorage/index.tsx | 6 +
inlong-website/src/utils/index.ts | 13 +
inlong-website/src/utils/metaData.ts | 66 +++++
11 files changed, 611 insertions(+), 178 deletions(-)
diff --git a/inlong-website/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx b/inlong-website/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx
index 7763b48..462d4d1 100644
--- a/inlong-website/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx
+++ b/inlong-website/src/components/AccessHelper/DataStorageEditor/DetailModal.tsx
@@ -26,7 +26,9 @@ import FormGenerator, {
FormItemProps,
FormGeneratorProps,
} from '@/components/FormGenerator';
+import { GetStorageFormFieldsType } from '@/utils/metaData';
import { getHiveForm, getHiveColumns } from '@/components/MetaData/StorageHive';
+import { getClickhouseForm, getClickhouseColumns } from '@/components/MetaData/StorageClickhouse';
export interface DetailModalProps extends ModalProps {
inlongGroupId: string;
@@ -77,6 +79,13 @@ const Comp: React.FC<DetailModalProps> = ({
fieldName: 'fieldName',
},
},
+ CLICK_HOUSE: {
+ columnsKey: 'clickHouseFieldList',
+ getColumns: getClickhouseColumns,
+ restMapping: {
+ fieldName: 'fieldName',
+ },
+ },
}[storageType];
}, [storageType]);
@@ -147,13 +156,19 @@ const Comp: React.FC<DetailModalProps> = ({
}, [modalProps.visible]);
const formContent = useMemo(() => {
- const map = {
+ const map: Record<string, GetStorageFormFieldsType> = {
HIVE: getHiveForm,
- // CLICK_HOUSE: getClickhouseForm,
+ CLICK_HOUSE: getClickhouseForm,
};
const item = map[storageType];
- return item(dataType, !!id, inlongGroupId, currentValues, form);
+ return item('form', {
+ dataType,
+ isEdit: !!id,
+ inlongGroupId,
+ currentValues,
+ form,
+ }) as FormItemProps[];
}, [storageType, dataType, inlongGroupId, id, currentValues, form]);
const onOk = async () => {
diff --git a/inlong-website/src/components/AccessHelper/FieldsConfig/dataFields.tsx b/inlong-website/src/components/AccessHelper/FieldsConfig/dataFields.tsx
index c30d105..9c2e724 100644
--- a/inlong-website/src/components/AccessHelper/FieldsConfig/dataFields.tsx
+++ b/inlong-website/src/components/AccessHelper/FieldsConfig/dataFields.tsx
@@ -275,13 +275,17 @@ export default (
value: 'HIVE',
},
{
+ label: 'CLICK_HOUSE',
+ value: 'CLICK_HOUSE',
+ },
+ {
label: i18n.t('components.AccessHelper.FieldsConfig.dataFields.AutoConsumption'),
value: 'AUTO_PUSH',
},
],
},
},
- ...['HIVE'].reduce(
+ ...['HIVE', 'CLICK_HOUSE'].reduce(
(acc, item) =>
acc.concat({
type: (
diff --git a/inlong-website/src/components/MetaData/DataSourcesFile.ts b/inlong-website/src/components/MetaData/DataSourcesFile.ts
index 63ce694..294d027 100644
--- a/inlong-website/src/components/MetaData/DataSourcesFile.ts
+++ b/inlong-website/src/components/MetaData/DataSourcesFile.ts
@@ -25,19 +25,19 @@ export const getCreateFormContent = () => {
const array = [
{
type: 'input',
- label: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.DataSourceIP'),
+ label: i18n.t('components.AccessHelper.DataSourceMetaData.File.DataSourceIP'),
name: 'ip',
rules: [
{ required: true },
{
pattern: rulesPattern.ip,
- message: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.IpRule'),
+ message: i18n.t('components.AccessHelper.DataSourceMetaData.File.IpRule'),
},
],
},
{
type: 'inputnumber',
- label: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.Port'),
+ label: i18n.t('components.AccessHelper.DataSourceMetaData.File.Port'),
name: 'port',
props: {
min: 1,
@@ -47,10 +47,10 @@ export const getCreateFormContent = () => {
},
{
type: 'input',
- label: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.FilePath'),
+ label: i18n.t('components.AccessHelper.DataSourceMetaData.File.FilePath'),
name: 'filePath',
rules: [{ required: true }],
- suffix: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.FillInTheAbsolutePath'),
+ suffix: i18n.t('components.AccessHelper.DataSourceMetaData.File.FillInTheAbsolutePath'),
},
];
@@ -59,17 +59,17 @@ export const getCreateFormContent = () => {
export const tableColumns: ColumnsType = [
{
- title: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.DataSourceIP'),
+ title: i18n.t('components.AccessHelper.DataSourceMetaData.File.DataSourceIP'),
dataIndex: 'ip',
width: 150,
},
{
- title: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.Port'),
+ title: i18n.t('components.AccessHelper.DataSourceMetaData.File.Port'),
dataIndex: 'port',
width: 120,
},
{
- title: i18n.t('components.AccessHelper.DataSourcesEditor.FileConfig.FilePath'),
+ title: i18n.t('components.AccessHelper.DataSourceMetaData.File.FilePath'),
dataIndex: 'filePath',
},
];
diff --git a/inlong-website/src/components/MetaData/SourceDataFields.ts b/inlong-website/src/components/MetaData/SourceDataFields.ts
index 899a42d..46b793b 100644
--- a/inlong-website/src/components/MetaData/SourceDataFields.ts
+++ b/inlong-website/src/components/MetaData/SourceDataFields.ts
@@ -18,6 +18,7 @@
*/
import i18n from '@/i18n';
+import { ColumnsItemProps } from '@/components/EditableTable';
export const fieldTypes = ['int', 'long', 'float', 'double', 'string', 'date', 'timestamp'].map(
item => ({
@@ -26,16 +27,16 @@ export const fieldTypes = ['int', 'long', 'float', 'double', 'string', 'date', '
}),
);
-export const sourceDataFields = [
+export const sourceDataFields: ColumnsItemProps[] = [
{
- title: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.SourceFieldName'),
+ title: i18n.t('components.AccessHelper.StorageMetaData.SourceFieldName'),
dataIndex: 'sourceFieldName',
initialValue: '',
rules: [
{ required: true },
{
pattern: /^[a-zA-Z][a-zA-Z0-9_]*$/,
- message: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.fieldNameRule'),
+ message: i18n.t('components.AccessHelper.StorageMetaData.SourceFieldNameRule'),
},
],
props: (text, record, idx, isNew) => ({
@@ -43,7 +44,7 @@ export const sourceDataFields = [
}),
},
{
- title: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.SourceFieldType'),
+ title: i18n.t('components.AccessHelper.StorageMetaData.SourceFieldType'),
dataIndex: 'sourceFieldType',
initialValue: fieldTypes[0].value,
type: 'select',
diff --git a/inlong-website/src/components/MetaData/StorageClickhouse.tsx b/inlong-website/src/components/MetaData/StorageClickhouse.tsx
new file mode 100644
index 0000000..8946abc
--- /dev/null
+++ b/inlong-website/src/components/MetaData/StorageClickhouse.tsx
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+ getColsFromFields,
+ GetStorageColumnsType,
+ GetStorageFormFieldsType,
+} from '@/utils/metaData';
+import i18n from '@/i18n';
+import { ColumnsType } from 'antd/es/table';
+import EditableTable from '@/components/EditableTable';
+import { excludeObject } from '@/utils';
+import { sourceDataFields } from './SourceDataFields';
+
+// CLICK_HOUSE targetType
+const clickhouseTargetTypes = [
+ 'string',
+ 'boolean',
+ 'byte',
+ 'short',
+ 'int',
+ 'long',
+ 'float',
+ 'double',
+ 'decimal',
+ // 'time',
+ // 'date',
+ // 'timestamp',
+ // 'map<string,string>',
+ // 'map<string,map<string,string>>',
+ // 'map<string,map<string,map<string,string>>>',
+].map(item => ({
+ label: item,
+ value: item,
+}));
+
+// Auto map to source fieldType
+export const fieldTypeMap = {
+ int: 'int',
+ long: 'long',
+ float: 'float',
+ double: 'double',
+ string: 'string',
+ date: 'string',
+ timestamp: 'string',
+};
+
+export const getClickhouseForm: GetStorageFormFieldsType = (
+ type: 'form' | 'col' = 'form',
+ { currentValues, inlongGroupId, isEdit, dataType } = {} as any,
+) => {
+ const fileds = [
+ {
+ name: 'clusterId',
+ type: 'select',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.Cluster'),
+ rules: [{ required: true }],
+ props: {
+ options: {
+ requestService: {
+ url: '/storage/listStorageCluster',
+ params: {
+ storageType: 'CLICK_HOUSE',
+ },
+ },
+ requestParams: {
+ formatResult: result =>
+ result.clickHouseClusterList?.map(item => ({
+ label: item.name,
+ value: item.id,
+ })),
+ },
+ requestAuto: isEdit,
+ },
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ },
+ {
+ name: 'dbName',
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.Db'),
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ _col: true,
+ },
+ {
+ name: 'tableName',
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.Table'),
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ _col: true,
+ },
+ {
+ name: 'flushInterval',
+ type: 'inputnumber',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.FlushInterval'),
+ initialValue: 1,
+ props: {
+ min: 1,
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ rules: [{ required: true }],
+ suffix: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.FlushIntervalUnit'),
+ },
+ {
+ name: 'packageSize',
+ type: 'inputnumber',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.PackageSize'),
+ initialValue: 1000,
+ props: {
+ min: 1,
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ rules: [{ required: true }],
+ suffix: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.PackageSizeUnit'),
+ },
+ {
+ name: 'retryTime',
+ type: 'inputnumber',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.RetryTime'),
+ initialValue: 3,
+ props: {
+ min: 1,
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ rules: [{ required: true }],
+ suffix: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.RetryTimeUnit'),
+ },
+ {
+ name: 'username',
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.Username'),
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ },
+ {
+ name: 'password',
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.Password'),
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ },
+ {
+ name: 'isDistribute',
+ type: 'radio',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.IsDistribute'),
+ initialValue: 0,
+ props: {
+ options: [
+ {
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.Yes'),
+ value: 1,
+ },
+ {
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.No'),
+ value: 0,
+ },
+ ],
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ rules: [{ required: true }],
+ },
+ {
+ name: 'partitionStrategy',
+ type: 'select',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.PartitionStrategy'),
+ initialValue: 'BALANCE',
+ rules: [{ required: true }],
+ props: {
+ options: [
+ {
+ label: 'BALANCE',
+ value: 'BALANCE',
+ },
+ {
+ label: 'RANDOM',
+ value: 'RANDOM',
+ },
+ {
+ label: 'HASH',
+ value: 'HASH',
+ },
+ ],
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ visible: values => values.isDistribute,
+ _col: true,
+ },
+ {
+ name: 'partitionFields',
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.PartitionFields'),
+ rules: [{ required: true }],
+ visible: values => values.isDistribute && values.partitionStrategy === 'HASH',
+ props: {
+ disabled: isEdit && [110, 130].includes(currentValues?.status),
+ },
+ },
+ {
+ name: 'clickHouseFieldList',
+ type: EditableTable,
+ props: {
+ size: 'small',
+ editing: ![110, 130].includes(currentValues?.status),
+ columns: getClickhouseColumns(dataType, currentValues),
+ },
+ },
+ ];
+
+ return type === 'col'
+ ? getColsFromFields(fileds)
+ : fileds.map(item => excludeObject(['_col'], item));
+};
+
+export const getClickhouseColumns: GetStorageColumnsType = (dataType, currentValues) => {
+ return [
+ ...sourceDataFields,
+ {
+ title: `ClickHouse${i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.FieldName')}`,
+ dataIndex: 'fieldName',
+ rules: [
+ { required: true },
+ {
+ pattern: /^[a-zA-Z][a-zA-Z0-9_]*$/,
+ message: i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.FieldNameRule'),
+ },
+ ],
+ props: (text, record, idx, isNew) => ({
+ disabled: [110, 130].includes(currentValues?.status as number) && !isNew,
+ }),
+ },
+ {
+ title: `ClickHouse${i18n.t('components.AccessHelper.StorageMetaData.Clickhouse.FieldType')}`,
+ dataIndex: 'fieldType',
+ initialValue: clickhouseTargetTypes[0].value,
+ type: 'select',
+ props: (text, record, idx, isNew) => ({
+ disabled: [110, 130].includes(currentValues?.status as number) && !isNew,
+ options: clickhouseTargetTypes,
+ }),
+ rules: [{ required: true }],
+ },
+ {
+ title: `ClickHouse${i18n.t(
+ 'components.AccessHelper.StorageMetaData.Clickhouse.FieldDescription',
+ )}`,
+ dataIndex: 'fieldComment',
+ props: (text, record, idx, isNew) => ({
+ disabled: [110, 130].includes(currentValues?.status as number) && !isNew,
+ }),
+ },
+ ];
+};
+
+export const clickhouseTableColumns = getClickhouseForm('col') as ColumnsType;
diff --git a/inlong-website/src/components/MetaData/StorageHive.tsx b/inlong-website/src/components/MetaData/StorageHive.tsx
index e1418db..f1f8682 100644
--- a/inlong-website/src/components/MetaData/StorageHive.tsx
+++ b/inlong-website/src/components/MetaData/StorageHive.tsx
@@ -19,9 +19,15 @@
import React from 'react';
import { Button, message } from 'antd';
+import {
+ getColsFromFields,
+ GetStorageColumnsType,
+ GetStorageFormFieldsType,
+} from '@/utils/metaData';
import EditableTable, { ColumnsItemProps } from '@/components/EditableTable';
import request from '@/utils/request';
import i18n from '@/i18n';
+import { excludeObject } from '@/utils';
import { sourceDataFields } from './SourceDataFields';
// hiveFieldTypes
@@ -48,15 +54,15 @@ const hiveFieldTypes = [
export const hiveTableColumns = [
{
- title: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.Db'),
+ title: i18n.t('components.AccessHelper.StorageMetaData.Hive.Db'),
dataIndex: 'dbName',
},
{
- title: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.TargetTable'),
+ title: i18n.t('components.AccessHelper.StorageMetaData.Hive.TargetTable'),
dataIndex: 'tableName',
},
{
- title: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.Username'),
+ title: i18n.t('components.AccessHelper.StorageMetaData.Hive.Username'),
dataIndex: 'username',
},
{
@@ -69,21 +75,18 @@ export const hiveTableColumns = [
},
];
-export const getHiveColumns = (
- dataType: string,
- currentValues: Record<string, unknown>,
-): ColumnsItemProps[] => [
+export const getHiveColumns: GetStorageColumnsType = (dataType, currentValues) => [
...([
...sourceDataFields,
{
- title: `HIVE${i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.FieldName')}`,
+ title: `HIVE${i18n.t('components.AccessHelper.StorageMetaData.Hive.FieldName')}`,
dataIndex: 'fieldName',
initialValue: '',
rules: [
{ required: true },
{
pattern: /^[a-zA-Z][a-zA-Z0-9_]*$/,
- message: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.fieldNameRule'),
+ message: i18n.t('components.AccessHelper.StorageMetaData.Hive.FieldNameRule'),
},
],
props: (text, record, idx, isNew) => ({
@@ -92,7 +95,7 @@ export const getHiveColumns = (
}),
},
{
- title: `HIVE${i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.FieldType')}`,
+ title: `HIVE${i18n.t('components.AccessHelper.StorageMetaData.Hive.FieldType')}`,
dataIndex: 'fieldType',
initialValue: hiveFieldTypes[0].value,
type: 'select',
@@ -103,128 +106,131 @@ export const getHiveColumns = (
rules: [{ required: true }],
},
{
- title: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.FieldDescription'),
+ title: i18n.t('components.AccessHelper.StorageMetaData.Hive.FieldDescription'),
dataIndex: 'fieldComment',
initialValue: '',
},
] as ColumnsItemProps[]),
];
-export const getHiveForm = (
- dataType: string,
- isEdit: boolean,
- inlongGroupId: string,
- currentValues: Record<string, unknown>,
- form,
-) => [
- {
- type: 'input',
- label: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.Db'),
- name: 'dbName',
- rules: [{ required: true }],
- props: {
- disabled: isEdit,
+export const getHiveForm: GetStorageFormFieldsType = (
+ type,
+ { currentValues, inlongGroupId, isEdit, dataType, form } = {} as any,
+) => {
+ const fileds = [
+ {
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Hive.Db'),
+ name: 'dbName',
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit,
+ },
},
- },
- {
- type: 'input',
- label: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.TargetTable'),
- name: 'tableName',
- rules: [{ required: true }],
- props: {
- disabled: isEdit,
+ {
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Hive.TargetTable'),
+ name: 'tableName',
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit,
+ },
},
- },
- {
- type: 'input',
- name: 'primaryPartition',
- label: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.First-levelPartition'),
- initialValue: 'dt',
- rules: [{ required: true }],
- props: {
- disabled: isEdit,
+ {
+ type: 'input',
+ name: 'primaryPartition',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Hive.First-levelPartition'),
+ initialValue: 'dt',
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit,
+ },
},
- },
- {
- type: 'input',
- label: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.Username'),
- name: 'username',
- rules: [{ required: true }],
- props: {
- disabled: isEdit,
+ {
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Hive.Username'),
+ name: 'username',
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit,
+ },
},
- },
- {
- type: 'password',
- label: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.Password'),
- name: 'password',
- rules: [{ required: true }],
- props: {
- disabled: isEdit,
- style: {
- maxWidth: 500,
+ {
+ type: 'password',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Hive.Password'),
+ name: 'password',
+ rules: [{ required: true }],
+ props: {
+ disabled: isEdit,
+ style: {
+ maxWidth: 500,
+ },
},
},
- },
- {
- type: 'input',
- label: 'JDBC URL',
- name: 'jdbcUrl',
- rules: [{ required: true }],
- props: {
- placeholder: 'jdbc:hive2://127.0.0.1:10000',
- disabled: isEdit,
- style: { width: 500 },
+ {
+ type: 'input',
+ label: 'JDBC URL',
+ name: 'jdbcUrl',
+ rules: [{ required: true }],
+ props: {
+ placeholder: 'jdbc:hive2://127.0.0.1:10000',
+ disabled: isEdit,
+ style: { width: 500 },
+ },
+ suffix: (
+ <Button
+ onClick={async () => {
+ const values = await form.validateFields(['username', 'password', 'jdbcUrl']);
+ const res = await request({
+ url: '/storage/query/testConnection',
+ method: 'POST',
+ data: values,
+ });
+ res
+ ? message.success(
+ i18n.t('components.AccessHelper.StorageMetaData.Hive.ConnectionSucceeded'),
+ )
+ : message.error(
+ i18n.t('components.AccessHelper.StorageMetaData.Hive.ConnectionFailed'),
+ );
+ }}
+ >
+ {i18n.t('components.AccessHelper.StorageMetaData.Hive.ConnectionTest')}
+ </Button>
+ ),
},
- suffix: (
- <Button
- onClick={async () => {
- const values = await form.validateFields(['username', 'password', 'jdbcUrl']);
- const res = await request({
- url: '/storage/query/testConnection',
- method: 'POST',
- data: values,
- });
- res
- ? message.success(
- i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionSucceeded'),
- )
- : message.error(
- i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionFailed'),
- );
- }}
- >
- {i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionTest')}
- </Button>
- ),
- },
- {
- type: 'input',
- label: 'HDFS DefaultFS',
- name: 'hdfsDefaultFs',
- rules: [{ required: true }],
- props: {
- placeholder: 'hdfs://127.0.0.1:9000',
- disabled: isEdit,
+ {
+ type: 'input',
+ label: 'HDFS DefaultFS',
+ name: 'hdfsDefaultFs',
+ rules: [{ required: true }],
+ props: {
+ placeholder: 'hdfs://127.0.0.1:9000',
+ disabled: isEdit,
+ },
},
- },
- {
- type: 'input',
- label: i18n.t('components.AccessHelper.DataStorageEditor.hiveConfig.WarehousePath'),
- name: 'warehouseDir',
- initialValue: '/user/hive/warehouse',
- props: {
- disabled: isEdit,
+ {
+ type: 'input',
+ label: i18n.t('components.AccessHelper.StorageMetaData.Hive.WarehousePath'),
+ name: 'warehouseDir',
+ initialValue: '/user/hive/warehouse',
+ props: {
+ disabled: isEdit,
+ },
},
- },
- {
- type: (
- <EditableTable
- size="small"
- columns={getHiveColumns(dataType, currentValues)}
- canDelete={(record, idx, isNew) => !isEdit || isNew}
- />
- ),
- name: 'hiveFieldList',
- },
-];
+ {
+ type: (
+ <EditableTable
+ size="small"
+ columns={getHiveColumns(dataType, currentValues)}
+ canDelete={(record, idx, isNew) => !isEdit || isNew}
+ />
+ ),
+ name: 'hiveFieldList',
+ },
+ ];
+
+ return type === 'col'
+ ? getColsFromFields(fileds)
+ : fileds.map(item => excludeObject(['_col'], item));
+};
diff --git a/inlong-website/src/locales/cn.json b/inlong-website/src/locales/cn.json
index d64d473..0903626 100644
--- a/inlong-website/src/locales/cn.json
+++ b/inlong-website/src/locales/cn.json
@@ -11,29 +11,50 @@
"basic.DeleteSuccess": "删除成功",
"basic.Status": "状态",
"basic.CreateTime": "创建时间",
- "components.AccessHelper.DataSourcesEditor.CreateModal.File": "文件",
- "components.AccessHelper.DataSourcesEditor.FileConfig.FillInTheAbsolutePath": "填写绝对路径",
- "components.AccessHelper.DataSourcesEditor.FileConfig.DataSourceIP": "数据源IP",
- "components.AccessHelper.DataSourcesEditor.FileConfig.FilePath": "⽂件路径",
- "components.AccessHelper.DataSourcesEditor.FileConfig.Port": "端口",
- "components.AccessHelper.DataSourcesEditor.FileConfig.IpRule": "请正确输入Ip地址",
"components.AccessHelper.DataSourcesEditor.NewDataSource": "新建数据源",
+ "components.AccessHelper.DataSourcesEditor.CreateModal.File": "文件",
+ "components.AccessHelper.DataSourceMetaData.File.FillInTheAbsolutePath": "填写绝对路径",
+ "components.AccessHelper.DataSourceMetaData.File.DataSourceIP": "数据源IP",
+ "components.AccessHelper.DataSourceMetaData.File.FilePath": "⽂件路径",
+ "components.AccessHelper.DataSourceMetaData.File.Port": "端口",
+ "components.AccessHelper.DataSourceMetaData.File.IpRule": "请正确输入Ip地址",
"components.AccessHelper.DataStorageEditor.Editor.AddTo": "添加",
- "components.AccessHelper.DataStorageEditor.hiveConfig.WarehousePath": "Warehouse路径",
- "components.AccessHelper.DataStorageEditor.hiveConfig.TargetTable": "目标表",
- "components.AccessHelper.DataStorageEditor.hiveConfig.fieldNameRule": "以英文字母开头，只能包含英文字母、数字、下划线",
- "components.AccessHelper.DataStorageEditor.hiveConfig.SourceFieldType": "源字段类型",
- "components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionSucceeded": "连接成功",
- "components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionTest": "连接测试",
- "components.AccessHelper.DataStorageEditor.hiveConfig.Db": "目标库",
- "components.AccessHelper.DataStorageEditor.hiveConfig.SourceFieldName": "源字段名",
- "components.AccessHelper.DataStorageEditor.hiveConfig.FieldName": "字段名",
- "components.AccessHelper.DataStorageEditor.hiveConfig.FieldType": "字段类型",
- "components.AccessHelper.DataStorageEditor.hiveConfig.First-levelPartition": "一级分区",
- "components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionFailed": "连接失败",
- "components.AccessHelper.DataStorageEditor.hiveConfig.Username": "用户名",
- "components.AccessHelper.DataStorageEditor.hiveConfig.Password": "Password",
- "components.AccessHelper.DataStorageEditor.hiveConfig.FieldDescription": "字段描述",
+ "components.AccessHelper.StorageMetaData.SourceFieldName": "源字段名",
+ "components.AccessHelper.StorageMetaData.SourceFieldType": "源字段类型",
+ "components.AccessHelper.StorageMetaData.SourceFieldNameRule": "以英文字母开头，只能包含英文字母、数字、下划线",
+ "components.AccessHelper.StorageMetaData.Hive.WarehousePath": "Warehouse路径",
+ "components.AccessHelper.StorageMetaData.Hive.TargetTable": "目标表",
+ "components.AccessHelper.StorageMetaData.Hive.FieldNameRule": "以英文字母开头，只能包含英文字母、数字、下划线",
+ "components.AccessHelper.StorageMetaData.Hive.ConnectionSucceeded": "连接成功",
+ "components.AccessHelper.StorageMetaData.Hive.ConnectionTest": "连接测试",
+ "components.AccessHelper.StorageMetaData.Hive.Db": "目标库",
+ "components.AccessHelper.StorageMetaData.Hive.FieldName": "字段名",
+ "components.AccessHelper.StorageMetaData.Hive.FieldType": "字段类型",
+ "components.AccessHelper.StorageMetaData.Hive.First-levelPartition": "一级分区",
+ "components.AccessHelper.StorageMetaData.Hive.ConnectionFailed": "连接失败",
+ "components.AccessHelper.StorageMetaData.Hive.Username": "用户名",
+ "components.AccessHelper.StorageMetaData.Hive.Password": "Password",
+ "components.AccessHelper.StorageMetaData.Hive.FieldDescription": "字段描述",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Cluster": "集群",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Db": "目标库名",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Table": "目标表名",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FlushInterval": "Flush间隔",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FlushIntervalUnit": "秒",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PackageSize": "打包条数",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PackageSizeUnit": "条",
+ "components.AccessHelper.StorageMetaData.Clickhouse.RetryTime": "写入失败重测次数",
+ "components.AccessHelper.StorageMetaData.Clickhouse.RetryTimeUnit": "次",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Username": "用户名",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Password": "密码",
+ "components.AccessHelper.StorageMetaData.Clickhouse.IsDistribute": "是否分布式表",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Yes": "是",
+ "components.AccessHelper.StorageMetaData.Clickhouse.No": "否",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PartitionStrategy": "分区策略",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PartitionFields": "分区字段",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldName": "字段名",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldNameRule": "以英文字母开头，只能包含英文字母、数字、下划线",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldType": "字段类型",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldDescription": "字段描述",
"components.AccessHelper.FieldsConfig.businessFields.Tube": "⾼吞吐（TUBE）",
"components.AccessHelper.FieldsConfig.businessFields.BusinessLabelName": "业务中文名称",
"components.AccessHelper.FieldsConfig.businessFields.Stripe/Second": "条/秒",
diff --git a/inlong-website/src/locales/en.json b/inlong-website/src/locales/en.json
index 28bb412..ce3b138 100644
--- a/inlong-website/src/locales/en.json
+++ b/inlong-website/src/locales/en.json
@@ -11,29 +11,50 @@
"basic.DeleteSuccess": "Delete Success",
"basic.Status": "Status",
"basic.CreateTime": "CreateTime",
- "components.AccessHelper.DataSourcesEditor.CreateModal.File": "File",
- "components.AccessHelper.DataSourcesEditor.FileConfig.FillInTheAbsolutePath": "Fill in the absolute path",
- "components.AccessHelper.DataSourcesEditor.FileConfig.DataSourceIP": "Data source IP",
- "components.AccessHelper.DataSourcesEditor.FileConfig.FilePath": "File path",
- "components.AccessHelper.DataSourcesEditor.FileConfig.Port": "Port",
- "components.AccessHelper.DataSourcesEditor.FileConfig.IpRule": "Please enter the IP address correctly",
"components.AccessHelper.DataSourcesEditor.NewDataSource": "New data source",
+ "components.AccessHelper.DataSourcesEditor.CreateModal.File": "File",
+ "components.AccessHelper.DataSourceMetaData.File.FillInTheAbsolutePath": "Fill in the absolute path",
+ "components.AccessHelper.DataSourceMetaData.File.DataSourceIP": "Data source IP",
+ "components.AccessHelper.DataSourceMetaData.File.FilePath": "File path",
+ "components.AccessHelper.DataSourceMetaData.File.Port": "Port",
+ "components.AccessHelper.DataSourceMetaData.File.IpRule": "Please enter the IP address correctly",
"components.AccessHelper.DataStorageEditor.Editor.AddTo": "Add",
- "components.AccessHelper.DataStorageEditor.hiveConfig.WarehousePath": "Warehouse path",
- "components.AccessHelper.DataStorageEditor.hiveConfig.TargetTable": "Target table",
- "components.AccessHelper.DataStorageEditor.hiveConfig.fieldNameRule": "At the beginning of English letters, only English letters, numbers, and underscores",
- "components.AccessHelper.DataStorageEditor.hiveConfig.SourceFieldType": "Source field type",
- "components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionSucceeded": "Connection succeeded",
- "components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionTest": "Connection test",
- "components.AccessHelper.DataStorageEditor.hiveConfig.Db": "DB",
- "components.AccessHelper.DataStorageEditor.hiveConfig.SourceFieldName": "Source field name",
- "components.AccessHelper.DataStorageEditor.hiveConfig.FieldName": "FieldName",
- "components.AccessHelper.DataStorageEditor.hiveConfig.FieldType": "FieldType",
- "components.AccessHelper.DataStorageEditor.hiveConfig.First-levelPartition": "First-level partition",
- "components.AccessHelper.DataStorageEditor.hiveConfig.ConnectionFailed": "Connection failed",
- "components.AccessHelper.DataStorageEditor.hiveConfig.Username": "Username",
- "components.AccessHelper.DataStorageEditor.hiveConfig.Password": "Password",
- "components.AccessHelper.DataStorageEditor.hiveConfig.FieldDescription": "Field description",
+ "components.AccessHelper.StorageMetaData.SourceFieldName": "SourceFieldName",
+ "components.AccessHelper.StorageMetaData.SourceFieldType": "SourceFieldType",
+ "components.AccessHelper.StorageMetaData.SourceFieldNameRule": "At the beginning of English letters, only English letters, numbers, and underscores",
+ "components.AccessHelper.StorageMetaData.Hive.WarehousePath": "Warehouse path",
+ "components.AccessHelper.StorageMetaData.Hive.TargetTable": "Target table",
+ "components.AccessHelper.StorageMetaData.Hive.FieldNameRule": "At the beginning of English letters, only English letters, numbers, and underscores",
+ "components.AccessHelper.StorageMetaData.Hive.ConnectionSucceeded": "Connection succeeded",
+ "components.AccessHelper.StorageMetaData.Hive.ConnectionTest": "Connection test",
+ "components.AccessHelper.StorageMetaData.Hive.Db": "DB",
+ "components.AccessHelper.StorageMetaData.Hive.FieldName": "FieldName",
+ "components.AccessHelper.StorageMetaData.Hive.FieldType": "FieldType",
+ "components.AccessHelper.StorageMetaData.Hive.First-levelPartition": "First-level partition",
+ "components.AccessHelper.StorageMetaData.Hive.ConnectionFailed": "Connection failed",
+ "components.AccessHelper.StorageMetaData.Hive.Username": "Username",
+ "components.AccessHelper.StorageMetaData.Hive.Password": "Password",
+ "components.AccessHelper.StorageMetaData.Hive.FieldDescription": "Field description",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Cluster": "Cluster",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Db": "Db",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Table": "Table",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FlushInterval": "FlushInterval",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FlushIntervalUnit": "S",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PackageSize": "PackageSize",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PackageSizeUnit": "items",
+ "components.AccessHelper.StorageMetaData.Clickhouse.RetryTime": "RetryTime",
+ "components.AccessHelper.StorageMetaData.Clickhouse.RetryTimeUnit": "items",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Username": "Username",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Password": "Password",
+ "components.AccessHelper.StorageMetaData.Clickhouse.IsDistribute": "IsDistribute",
+ "components.AccessHelper.StorageMetaData.Clickhouse.Yes": "Yes",
+ "components.AccessHelper.StorageMetaData.Clickhouse.No": "No",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PartitionStrategy": "PartitionStrategy",
+ "components.AccessHelper.StorageMetaData.Clickhouse.PartitionFields": "PartitionFields",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldName": "FieldName",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldNameRule": "At the beginning of English letters, only English letters, numbers, and underscores",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldType": "FieldType",
+ "components.AccessHelper.StorageMetaData.Clickhouse.FieldDescription": "FieldDescription",
"components.AccessHelper.FieldsConfig.businessFields.Tube": "Tube",
"components.AccessHelper.FieldsConfig.businessFields.BusinessLabelName": "Business Label",
"components.AccessHelper.FieldsConfig.businessFields.Stripe/Second": "Stripe / S",
diff --git a/inlong-website/src/pages/AccessDetail/DataStorage/index.tsx b/inlong-website/src/pages/AccessDetail/DataStorage/index.tsx
index 9346f63..5576504 100644
--- a/inlong-website/src/pages/AccessDetail/DataStorage/index.tsx
+++ b/inlong-website/src/pages/AccessDetail/DataStorage/index.tsx
@@ -25,6 +25,7 @@ import { useRequest } from '@/hooks';
import i18n from '@/i18n';
import { DataStorageDetailModal } from '@/components/AccessHelper';
import { hiveTableColumns } from '@/components/MetaData/StorageHive';
+import { clickhouseTableColumns } from '@/components/MetaData/StorageClickhouse';
import request from '@/utils/request';
import { CommonInterface } from '../common';
import { genStatusTag } from './status';
@@ -48,6 +49,10 @@ const getFilterFormContent = defaultValues => [
label: 'HIVE',
value: 'HIVE',
},
+ {
+ label: 'CLICK_HOUSE',
+ value: 'CLICK_HOUSE',
+ },
],
},
},
@@ -160,6 +165,7 @@ const Comp: React.FC<Props> = ({ inlongGroupId }) => {
const columnsMap = {
HIVE: hiveTableColumns,
+ CLICK_HOUSE: clickhouseTableColumns,
};
const createContent = useMemo(
diff --git a/inlong-website/src/utils/index.ts b/inlong-website/src/utils/index.ts
index 7fe1305..84549f4 100644
--- a/inlong-website/src/utils/index.ts
+++ b/inlong-website/src/utils/index.ts
@@ -161,6 +161,19 @@ export function pickObject(keys = [], sourceObj) {
}, {});
}
+export function excludeObject(keys = [], sourceObj: Record<string, unknown>) {
+ const set = new Set(keys);
+ return Object.entries(sourceObj).reduce((acc, [key, value]) => {
+ if (!set.has(key)) {
+ return {
+ ...acc,
+ [key]: value,
+ };
+ }
+ return acc;
+ }, {});
+}
+
/**
* Exclude parts from the object array to form a new array
* @param keys
diff --git a/inlong-website/src/utils/metaData.ts b/inlong-website/src/utils/metaData.ts
new file mode 100644
index 0000000..4b388d3
--- /dev/null
+++ b/inlong-website/src/utils/metaData.ts
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FormItemProps } from '@/components/FormGenerator';
+import { ColumnsType } from 'antd/es/table';
+import { ColumnsItemProps } from '@/components/EditableTable';
+
+export type GetStorageColumnsType = (
+ dataType: string,
+ currentValues?: Record<string, unknown>,
+) => ColumnsItemProps[];
+
+export type GetStorageFormFieldsType = (
+ type: 'form' | 'col',
+ {
+ currentValues,
+ inlongGroupId,
+ isEdit,
+ dataType,
+ form,
+ }?: {
+ currentValues?: Record<string, any>;
+ inlongGroupId?: string;
+ isEdit?: boolean;
+ dataType?: string;
+ form?: any;
+ },
+) => FormItemProps[] | ColumnsType;
+
+interface FieldConfigItem extends FormItemProps {
+ _col?: boolean | Record<string, unknown>;
+}
+
+export const getColsFromFields = (fieldsConfig: FieldConfigItem[]): ColumnsType => {
+ return fieldsConfig
+ .filter(item => item._col)
+ .map(item => {
+ let output = {
+ title: item.label,
+ dataIndex: item.name,
+ };
+ if (typeof item._col === 'object') {
+ output = {
+ ...output,
+ ...item._col,
+ };
+ }
+ return output;
+ });
+};