You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by he...@apache.org on 2022/08/12 01:46:29 UTC
[inlong] branch master updated: [INLONG-5474][Manager][Agent] Add metadata parameter (#5475)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5c6690cfe [INLONG-5474][Manager][Agent] Add metadata parameter (#5475)
5c6690cfe is described below
commit 5c6690cfe7012ffb2f6bce32189a59e7d9c1d49a
Author: ganfengtan <Ga...@users.noreply.github.com>
AuthorDate: Fri Aug 12 09:46:23 2022 +0800
[INLONG-5474][Manager][Agent] Add metadata parameter (#5475)
---
.../java/org/apache/inlong/agent/constant/JobConstants.java | 1 +
.../src/main/java/org/apache/inlong/agent/pojo/FileJob.java | 11 +++++++++++
.../main/java/org/apache/inlong/agent/pojo/JobProfileDto.java | 6 +++++-
.../apache/inlong/manager/pojo/source/file/FileSource.java | 3 +++
.../apache/inlong/manager/pojo/source/file/FileSourceDTO.java | 4 ++++
.../inlong/manager/pojo/source/file/FileSourceRequest.java | 3 +++
6 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/JobConstants.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/JobConstants.java
index 27330859d..04db5b097 100755
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/JobConstants.java
+++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/JobConstants.java
@@ -55,6 +55,7 @@ public class JobConstants extends CommonConstants {
public static final String JOB_FILE_CONTENT_COLLECT_TYPE = "job.fileJob.contentCollectType";
public static final String JOB_FILE_META_ENV_LIST = "job.fileJob.envList";
public static final String JOB_FILE_DATA_SOURCE_COLUMN_SEPARATOR = "job.fileJob.dataSeparator";
+ public static final String JOB_FILE_META_FILTER_BY_LABELS = "job.fileJob.filterMetaByLabels";
//Binlog job
public static final String JOB_DATABASE_USER = "job.binlogJob.user";
diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java
index a8bbfceb4..bde908dbd 100644
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java
+++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java
@@ -45,6 +45,10 @@ public class FileJob {
private String dataSeparator;
+ private Map<String, String> filterMetaByLabels;
+
+ private Map<String, Object> properties;
+
@Data
public static class Dir {
@@ -100,6 +104,13 @@ public class FileJob {
// Column separator of data source
private String dataSeparator;
+
+ // Metadata filters by label, special parameters for K8S
+ private Map<String, String> filterMetaByLabels;
+
+ // Properties for File
+ private Map<String, Object> properties;
+
}
}
diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/JobProfileDto.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/JobProfileDto.java
index 0679a384f..aa53b8346 100644
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/JobProfileDto.java
+++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/JobProfileDto.java
@@ -107,7 +107,7 @@ public class JobProfileDto {
fileJob.setCollectType(fileJobTaskConfig.getCollectType());
fileJob.setContentCollectType(fileJobTaskConfig.getContentCollectType());
fileJob.setDataSeparator(fileJobTaskConfig.getDataSeparator());
-
+ fileJob.setProperties(fileJobTaskConfig.getProperties());
if (fileJobTaskConfig.getTimeOffset() != null) {
fileJob.setTimeOffset(fileJobTaskConfig.getTimeOffset());
}
@@ -128,6 +128,10 @@ public class JobProfileDto {
if (null != fileJobTaskConfig.getMetaFields()) {
fileJob.setMetaFields(fileJob.getMetaFields());
}
+
+ if (null != fileJobTaskConfig.getFilterMetaByLabels()) {
+ fileJob.setFilterMetaByLabels(fileJobTaskConfig.getFilterMetaByLabels());
+ }
return fileJob;
}
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSource.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSource.java
index 61577bf58..f90c09bce 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSource.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSource.java
@@ -79,6 +79,9 @@ public class FileSource extends StreamSource {
+ " Json format, set this parameter to json ")
private String dataContentStyle;
+ @ApiModelProperty("Metadata filters by label, special parameters for K8S")
+ private Map<String, String> filterMetaByLabels;
+
public FileSource() {
this.setSourceType(SourceType.FILE);
}
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceDTO.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceDTO.java
index f481b77d4..815452751 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceDTO.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceDTO.java
@@ -82,6 +82,9 @@ public class FileSourceDTO {
@ApiModelProperty("Column separator of data source ")
private String dataSeparator;
+ @ApiModelProperty("Metadata filters by label, special parameters for K8S")
+ private Map<String, String> filterMetaByLabels;
+
public static FileSourceDTO getFromRequest(@NotNull FileSourceRequest fileSourceRequest) {
return FileSourceDTO.builder()
.clusterTag(fileSourceRequest.getClusterTag())
@@ -91,6 +94,7 @@ public class FileSourceDTO {
.contentCollectType(fileSourceRequest.getContentCollectType())
.envList(fileSourceRequest.getEnvList())
.dataContentStyle(fileSourceRequest.getDataContentStyle())
+ .filterMetaByLabels(fileSourceRequest.getFilterMetaByLabels())
.metaFields(fileSourceRequest.getMetaFields())
.timeOffset(fileSourceRequest.getTimeOffset())
.properties(fileSourceRequest.getProperties())
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceRequest.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceRequest.java
index c68bce85e..90907125a 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceRequest.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/file/FileSourceRequest.java
@@ -74,6 +74,9 @@ public class FileSourceRequest extends SourceRequest {
+ " Json format, set this parameter to json ")
private String dataContentStyle;
+ @ApiModelProperty("Metadata filters by label, special parameters for K8S")
+ private Map<String, String> filterMetaByLabels;
+
public FileSourceRequest() {
this.setSourceType(SourceType.FILE);
this.setSerializationType(DataFormat.CSV.getName());