You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by GitBox <gi...@apache.org> on 2022/01/10 06:48:36 UTC
[GitHub] [incubator-inlong] EMsnap commented on a change in pull request #2110: [INLONG-2109][manager] support db SQL collect
EMsnap commented on a change in pull request #2110:
URL: https://github.com/apache/incubator-inlong/pull/2110#discussion_r779388916
##########
File path: inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/dtos/JobProfileDto.java
##########
@@ -17,31 +17,49 @@
package org.apache.inlong.agent.plugin.fetcher.dtos;
+import static org.apache.inlong.agent.plugin.fetcher.constants.FetcherConstants.AGENT_MANAGER_VIP_HTTP_HOST;
+import static org.apache.inlong.agent.plugin.fetcher.constants.FetcherConstants.AGENT_MANAGER_VIP_HTTP_PORT;
+
+import com.google.gson.Gson;
import lombok.Data;
+import org.apache.inlong.agent.conf.AgentConfiguration;
+import org.apache.inlong.agent.conf.TriggerProfile;
@Data
public class JobProfileDto {
+
+ private static final Gson GSON = new Gson();
private Job job;
private Proxy proxy;
+ public static final String DEFAULT_TRIGGER = "org.apache.inlong.agent.plugin.trigger.DirectoryTrigger";
+ public static final String DEFAULT_CHANNEL = "org.apache.inlong.agent.plugin.channel.MemoryChannel";
+ public static final String TDM_JOB = "TDM_JOB";
Review comment:
tdm -> manager
##########
File path: inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/dtos/SqlJobProfileDto.java
##########
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.fetcher.dtos;
+
+import static org.apache.inlong.agent.plugin.fetcher.constants.FetcherConstants.AGENT_MANAGER_VIP_HTTP_HOST;
+import static org.apache.inlong.agent.plugin.fetcher.constants.FetcherConstants.AGENT_MANAGER_VIP_HTTP_PORT;
+
+import com.google.gson.Gson;
+import lombok.Data;
+import org.apache.inlong.agent.conf.AgentConfiguration;
+import org.apache.inlong.agent.conf.JobProfile;
+import org.apache.inlong.agent.conf.TriggerProfile;
+
+@Data
+public class SqlJobProfileDto {
+
+ private static final Gson GSON = new Gson();
+ private Job job;
+ private Proxy proxy;
+
+ public static final String SQL_JOB = "SQL_JOB";
+ public static final String DEFAULT_CHANNEL = "org.apache.inlong.agent.plugin.channel.MemoryChannel";
+ public static final String DEFAULT_BUS_SINK = "org.apache.inlong.agent.plugin.sinks.ProxySink";
+ public static final String DEFAULT_SOURCE = "org.apache.inlong.agent.plugin.sources.TextFileSource";
+
+ @Data
+ public static class Running {
+
+ private String core;
+ }
+
+ @Data
+ public static class Thread {
+
+ private Running running;
+ }
+
+ @Data
+ public static class Job {
+
+ private int id;
+ //private Thread thread;
+ private String name;
+ private String source;
+ private String sink;
+ private String channel;
+ //private String pattern;
+ //private String op;
+ //private String cycleUnit;
+ //private String timeOffset;
+ //private String deliveryTime;
+ //private String addictiveString;
+ private String ip;
+ private Integer port;
+ private String dbName;
+ private String user;
+ private String password;
+ private String sqlStatement;
+ private Integer totalLimit;
+ private Integer onceLimit;
+ private Integer timeLimit;
+ private Integer retryTimes;
+ }
+
+ @Data
+ public static class Manager {
+
+ private String port;
+ private String host;
+ }
+
+ @Data
+ public static class Proxy {
+
+ private String inlongGroupId;
+ private String inlongStreamId;
+ private Manager manager;
+ }
+
+ private static Job getJob(DbCollectorTask task) {
+ Job job = new Job();
+ job.setId(Integer.parseInt(task.getId()));
+ job.setName(SQL_JOB);
+ job.setSource(DEFAULT_SOURCE);
+ job.setSink(DEFAULT_BUS_SINK);
+ job.setChannel(DEFAULT_CHANNEL);
+ //job.setTimeOffset(dataConfigs.getTimeOffset());
+ //job.setOp(dataConfigs.getOp());
+ //job.setDeliveryTime(dataConfigs.getDeliveryTime());
+ //if (!dataConfigs.getAdditionalAttr().isEmpty()) {
+ // job.setAddictiveString(dataConfigs.getAdditionalAttr());
Review comment:
remove useless Annotation pls
##########
File path: inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/MiniAgent.java
##########
@@ -46,7 +46,7 @@ public void start() throws Exception {
}
public void submitJob(JobProfile profile) {
- manager.getJobManager().submitJobProfile(profile);
+ manager.getJobManager().submitFileJobProfile(profile);
Review comment:
method name should be submitFileJob?
##########
File path: inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/dtos/SqlJobProfileDto.java
##########
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.fetcher.dtos;
+
+import static org.apache.inlong.agent.plugin.fetcher.constants.FetcherConstants.AGENT_MANAGER_VIP_HTTP_HOST;
+import static org.apache.inlong.agent.plugin.fetcher.constants.FetcherConstants.AGENT_MANAGER_VIP_HTTP_PORT;
+
+import com.google.gson.Gson;
+import lombok.Data;
+import org.apache.inlong.agent.conf.AgentConfiguration;
+import org.apache.inlong.agent.conf.JobProfile;
+import org.apache.inlong.agent.conf.TriggerProfile;
+
+@Data
+public class SqlJobProfileDto {
+
+ private static final Gson GSON = new Gson();
+ private Job job;
+ private Proxy proxy;
+
+ public static final String SQL_JOB = "SQL_JOB";
+ public static final String DEFAULT_CHANNEL = "org.apache.inlong.agent.plugin.channel.MemoryChannel";
+ public static final String DEFAULT_BUS_SINK = "org.apache.inlong.agent.plugin.sinks.ProxySink";
+ public static final String DEFAULT_SOURCE = "org.apache.inlong.agent.plugin.sources.TextFileSource";
+
+ @Data
+ public static class Running {
+
+ private String core;
+ }
+
+ @Data
+ public static class Thread {
+
+ private Running running;
+ }
+
+ @Data
+ public static class Job {
+
+ private int id;
+ //private Thread thread;
+ private String name;
+ private String source;
+ private String sink;
+ private String channel;
+ //private String pattern;
+ //private String op;
+ //private String cycleUnit;
+ //private String timeOffset;
+ //private String deliveryTime;
Review comment:
remove useless annotations pls
##########
File path: inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/dtos/DbCollectorTask.java
##########
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.fetcher.dtos;
+
+import lombok.Data;
+
+@Data
+public class DbCollectorTask {
+
+ private String id;
+ private Integer type;
+ private Integer dbType;
+ private String ip;
+ private Integer port;
+ private String dbName;
+ private String user;
Review comment:
add a filed named separator to speparator db fields when sending data to sort
note that manager should be modified correspondingly
##########
File path: inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/dtos/JobProfileDto.java
##########
@@ -60,14 +78,63 @@
@Data
public static class Manager {
+
private String port;
private String host;
}
-
+
@Data
public static class Proxy {
+
private String inlongGroupId;
private String inlongStreamId;
private Manager manager;
}
+
+ private static Job getJob(DataConfig dataConfigs) {
+ Job job = new Job();
+ Dir dir = new Dir();
+ dir.setPattern(dataConfigs.getDataName());
+ job.setDir(dir);
+ job.setTrigger(DEFAULT_TRIGGER);
+ job.setChannel(DEFAULT_CHANNEL);
+ job.setName(TDM_JOB);
+ job.setSource(DEFAULT_SOURCE);
+ job.setSink(DEFAULT_BUS_SINK);
Review comment:
bus -> proxy
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org