You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by GitBox <gi...@apache.org> on 2022/08/17 01:52:50 UTC
[GitHub] [inlong] vernedeng commented on a diff in pull request #5389: [INLONG-4976][Feature][Manager] Support Kafka
vernedeng commented on code in PR #5389:
URL: https://github.com/apache/inlong/pull/5389#discussion_r946413815
##########
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/kafka/InlongKafkaDTO.java:
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.group.kafka;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.swagger.annotations.ApiModel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.exceptions.BusinessException;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * Inlong group info for Kafka
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@ApiModel("Inlong group info for Kafka")
+public class InlongKafkaDTO {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); // thread safe
+
+ private int tmp;
+
+ /**
+ * Get the dto instance from the request
+ */
+ public static InlongKafkaDTO getFromRequest(InlongKafkaRequest request) {
+ return InlongKafkaDTO.builder()
+ .tmp(request.getTmp())
+ .build();
+ }
+
+ /**
+ * Get the dto instance from the JSON string.
+ */
+ public static InlongKafkaDTO getFromJson(@NotNull String extParams) {
+ try {
+ OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
Review Comment:
ditto
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongKafkaOperator.java:
##########
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.group;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.inlong.manager.common.consts.MQType;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.exceptions.BusinessException;
+import org.apache.inlong.manager.common.util.CommonBeanUtils;
+import org.apache.inlong.manager.dao.entity.InlongGroupEntity;
+import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.pojo.group.InlongGroupRequest;
+import org.apache.inlong.manager.pojo.group.InlongGroupTopicInfo;
+import org.apache.inlong.manager.pojo.group.kafka.InlongKafkaDTO;
+import org.apache.inlong.manager.pojo.group.kafka.InlongKafkaInfo;
+import org.apache.inlong.manager.pojo.group.kafka.InlongKafkaRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Inlong group operator for Kafka.
+ */
+@Service
+public class InlongKafkaOperator extends AbstractGroupOperator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InlongKafkaOperator.class);
+
+ @Autowired
+ private ObjectMapper objectMapper;
+
+ @Override
+ public Boolean accept(String mqType) {
+ return getMQType().equals(mqType) || MQType.KAFKA.equals(mqType);
+ }
+
+ @Override
+ public String getMQType() {
+ return MQType.KAFKA;
+ }
+
+ @Override
+ public InlongGroupInfo getFromEntity(InlongGroupEntity entity) {
+ if (entity == null) {
+ throw new BusinessException(ErrorCodeEnum.GROUP_NOT_FOUND);
+ }
+
+ InlongKafkaInfo groupInfo = new InlongKafkaInfo();
+ CommonBeanUtils.copyProperties(entity, groupInfo);
+
+ if (StringUtils.isNotBlank(entity.getExtParams())) {
+ InlongKafkaDTO dto = InlongKafkaDTO.getFromJson(entity.getExtParams());
+ CommonBeanUtils.copyProperties(dto, groupInfo);
+ }
+
+ return groupInfo;
+ }
+
+ @Override
+ protected void setTargetEntity(InlongGroupRequest request, InlongGroupEntity targetEntity) {
+ InlongKafkaRequest kafkaRequest = (InlongKafkaRequest) request;
+
+ //TODO check kafka params
+
+ CommonBeanUtils.copyProperties(kafkaRequest, targetEntity, true);
+ try {
+ InlongKafkaDTO dto = InlongKafkaDTO.getFromRequest(kafkaRequest);
+ targetEntity.setExtParams(objectMapper.writeValueAsString(dto));
+ } catch (Exception e) {
+ throw new BusinessException(ErrorCodeEnum.SOURCE_INFO_INCORRECT.getMessage() + ": " + e.getMessage());
+ }
+ LOGGER.info("success set entity for inlong group with Pulsar");
+ }
+
+ @Override
+ public InlongGroupTopicInfo getTopic(InlongGroupInfo groupInfo) {
+ InlongGroupTopicInfo topicInfo = super.getTopic(groupInfo);
+ // TODO add cache for cluster info
+ // pulsar topic corresponds to the inlong stream one-to-one
+ // topicInfo.setDsTopicList(streamService.getTopicList(groupInfo.getInlongGroupId()));
+ // commonOperateService.getSpecifiedParam(InlongConstants.TUBE_MASTER_URL);
+ // groupInfo.setTenant();
+ // groupInfo.setAdminUrl();
+ // groupInfo.setServiceUrl();
Review Comment:
seems like code of pulsar operator， please remove them
##########
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/cluster/kafka/KafkaClusterDTO.java:
##########
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.cluster.kafka;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.exceptions.BusinessException;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * Kafka cluster info
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@ApiModel("Kafka cluster info")
+public class KafkaClusterDTO {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); // thread safe
+
+ @ApiModelProperty(value = "Kafka admin URL, such as: http://127.0.0.1:8080",
+ notes = "Pulsar service URL is the 'url' field of the cluster")
+ private String adminUrl;
+
+ /**
+ * Get the dto instance from the request
+ */
+ public static KafkaClusterDTO getFromRequest(KafkaClusterRequest request) {
+ return KafkaClusterDTO.builder()
+ .adminUrl(request.getAdminUrl())
+ .build();
+ }
+
+ /**
+ * Get the dto instance from the JSON string.
+ */
+ public static KafkaClusterDTO getFromJson(@NotNull String extParams) {
+ try {
+ OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
Review Comment:
Better to configure ObjectMapper once at the very beginning, instead of each time you call _getFromJson_
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongKafkaOperator.java:
##########
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.group;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.inlong.manager.common.consts.MQType;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.exceptions.BusinessException;
+import org.apache.inlong.manager.common.util.CommonBeanUtils;
+import org.apache.inlong.manager.dao.entity.InlongGroupEntity;
+import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.pojo.group.InlongGroupRequest;
+import org.apache.inlong.manager.pojo.group.InlongGroupTopicInfo;
+import org.apache.inlong.manager.pojo.group.kafka.InlongKafkaDTO;
+import org.apache.inlong.manager.pojo.group.kafka.InlongKafkaInfo;
+import org.apache.inlong.manager.pojo.group.kafka.InlongKafkaRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Inlong group operator for Kafka.
+ */
+@Service
+public class InlongKafkaOperator extends AbstractGroupOperator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InlongKafkaOperator.class);
+
+ @Autowired
+ private ObjectMapper objectMapper;
+
+ @Override
+ public Boolean accept(String mqType) {
+ return getMQType().equals(mqType) || MQType.KAFKA.equals(mqType);
Review Comment:
getMQType() is exactly same to MQType.KAFKA, I think the latter one is redundent
##########
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupTopicInfo.java:
##########
@@ -53,4 +53,7 @@ public class InlongGroupTopicInfo {
@ApiModelProperty(value = "Pulsar admin URL")
private String pulsarAdminUrl;
+ @ApiModelProperty(value = "Pulsar admin URL")
Review Comment:
kafka, no pulsar
##########
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/kafka/InlongKafkaRequest.java:
##########
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.group.kafka;
+
+import io.swagger.annotations.ApiModel;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import org.apache.inlong.manager.common.consts.MQType;
+import org.apache.inlong.manager.common.util.JsonTypeDefine;
+import org.apache.inlong.manager.pojo.group.InlongGroupRequest;
+
+/**
+ * Inlong group request for Kafka
+ */
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@ApiModel("Inlong group request for Kafka")
+@JsonTypeDefine(value = MQType.KAFKA)
+public class InlongKafkaRequest extends InlongGroupRequest {
+
+ private int tmp;
Review Comment:
ditto
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/kafka/KafkaOperator.java:
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.kafka;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.inlong.manager.pojo.cluster.kafka.KafkaClusterInfo;
+import org.apache.inlong.manager.service.cluster.InlongClusterServiceImpl;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.CreateTopicsResult;
+import org.apache.kafka.clients.admin.DeleteTopicsResult;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * kafka operator, supports creating topics and creating subscription.
+ */
+@Service
+public class KafkaOperator {
Review Comment:
easy to confuse with _InlongKafkaOperator_
##########
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ClusterType.java:
##########
@@ -26,4 +26,6 @@ public class ClusterType {
public static final String TUBEMQ = "TUBEMQ";
public static final String PULSAR = "PULSAR";
public static final String DATAPROXY = "DATAPROXY";
+
Review Comment:
no need to introduce an empty line
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/kafka/KafkaOperator.java:
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.kafka;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.inlong.manager.pojo.cluster.kafka.KafkaClusterInfo;
+import org.apache.inlong.manager.service.cluster.InlongClusterServiceImpl;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.CreateTopicsResult;
+import org.apache.kafka.clients.admin.DeleteTopicsResult;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * kafka operator, supports creating topics and creating subscription.
+ */
+@Service
+public class KafkaOperator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InlongClusterServiceImpl.class);
+
+ /**
+ * Create Kafka topic
+ */
+ public void createTopic(KafkaClusterInfo kafkaClusterInfo, String topicName)
+ throws InterruptedException, ExecutionException {
+ AdminClient adminClient = KafkaUtils.getAdminClient(kafkaClusterInfo);
+ NewTopic topic = new NewTopic(topicName,
+ kafkaClusterInfo.getNumPartitions(),
+ kafkaClusterInfo.getReplicationFactor());
+ CreateTopicsResult result = adminClient.createTopics(Collections.singletonList(topic));
+ // To prevent the client from disconnecting too quickly and causing the Topic to not be created successfully
+ Thread.sleep(500);
+ LOGGER.info("success to create kafka topic={}, with={} numPartitions",
+ topicName,
+ result.numPartitions(topicName).get());
+ }
+
+ /**
+ * Force delete Pulsar topic
+ */
+ public void forceDeleteTopic(KafkaClusterInfo kafkaClusterInfo, String topicName) {
+ AdminClient adminClient = KafkaUtils.getAdminClient(kafkaClusterInfo);
+ DeleteTopicsResult result = adminClient.deleteTopics(Collections.singletonList(topicName));
+ LOGGER.info("success to delete topic={}", topicName);
+ }
+
+ public boolean topicIsExists(KafkaClusterInfo kafkaClusterInfo, String topic)
+ throws ExecutionException, InterruptedException {
+ AdminClient adminClient = KafkaUtils.getAdminClient(kafkaClusterInfo);
+ Set<String> topicList = adminClient.listTopics().names().get();
+ return topicList.contains(topic);
+ }
+
+ public void createSubscription(KafkaClusterInfo kafkaClusterInfo, String subscription) {
+
+ KafkaConsumer kafkaConsumer = KafkaUtils.createKafkaConsumer(kafkaClusterInfo);
+ // subscription
+ kafkaConsumer.subscribe(Collections.singletonList(subscription));
+ }
+
+ public boolean subscriptionIsExists(KafkaClusterInfo kafkaClusterInfo, String topic) {
+ KafkaConsumer consumer = KafkaUtils.createKafkaConsumer(kafkaClusterInfo);
+ try {
+ Map<String, List<PartitionInfo>> topics = consumer.listTopics();
+ List<PartitionInfo> partitions = topics.get(topic);
+ if (partitions == null) {
+ LOGGER.info("subscription is exist");
Review Comment:
It seems like subscription is **_not_** exist, instead of exist
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/kafka/KafkaOperator.java:
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.kafka;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.inlong.manager.pojo.cluster.kafka.KafkaClusterInfo;
+import org.apache.inlong.manager.service.cluster.InlongClusterServiceImpl;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.CreateTopicsResult;
+import org.apache.kafka.clients.admin.DeleteTopicsResult;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+/**
+ * kafka operator, supports creating topics and creating subscription.
+ */
+@Service
+public class KafkaOperator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InlongClusterServiceImpl.class);
+
+ /**
+ * Create Kafka topic
+ */
+ public void createTopic(KafkaClusterInfo kafkaClusterInfo, String topicName)
+ throws InterruptedException, ExecutionException {
+ AdminClient adminClient = KafkaUtils.getAdminClient(kafkaClusterInfo);
+ NewTopic topic = new NewTopic(topicName,
+ kafkaClusterInfo.getNumPartitions(),
+ kafkaClusterInfo.getReplicationFactor());
+ CreateTopicsResult result = adminClient.createTopics(Collections.singletonList(topic));
+ // To prevent the client from disconnecting too quickly and causing the Topic to not be created successfully
+ Thread.sleep(500);
+ LOGGER.info("success to create kafka topic={}, with={} numPartitions",
+ topicName,
+ result.numPartitions(topicName).get());
+ }
+
+ /**
+ * Force delete Pulsar topic
Review Comment:
please check the comments, replace _pulsar_ to _kafka_
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/consumption/ConsumptionCompleteProcessListener.java:
##########
@@ -91,6 +91,9 @@ public ListenerResult listen(WorkflowContext context) throws WorkflowListenerExc
return ListenerResult.success("Create TubeMQ consumer group successful");
} else if (MQType.PULSAR.equals(mqType) || MQType.TDMQ_PULSAR.equals(mqType)) {
this.createPulsarSubscription(entity);
+ } else if (MQType.KAFKA.equals(mqType)) {
+ //TODO add kakfa
+
Review Comment:
nothing to do with kafka type
##########
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/kafka/InlongKafkaInfo.java:
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.group.kafka;
+
+import io.swagger.annotations.ApiModel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import lombok.experimental.SuperBuilder;
+import org.apache.inlong.manager.common.consts.MQType;
+import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.common.util.CommonBeanUtils;
+import org.apache.inlong.manager.common.util.JsonTypeDefine;
+
+/**
+ * Inlong group info for Kafka
+ */
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@JsonTypeDefine(value = MQType.KAFKA)
+@ApiModel("Inlong group info for Kafka")
+public class InlongKafkaInfo extends InlongGroupInfo {
+ private int tmp;
Review Comment:
“tmp” is not a good name, please rename it and add some comment
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/kafka/KafkaResourceOperators.java:
##########
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.kafka;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.common.consts.MQType;
+import org.apache.inlong.manager.common.enums.ClusterType;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
+import org.apache.inlong.manager.common.util.Preconditions;
+import org.apache.inlong.manager.pojo.cluster.ClusterInfo;
+import org.apache.inlong.manager.pojo.cluster.kafka.KafkaClusterInfo;
+import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamBriefInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.service.cluster.InlongClusterService;
+import org.apache.inlong.manager.service.core.ConsumptionService;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperator;
+import org.apache.inlong.manager.service.stream.InlongStreamService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * kafka for create kafka Topic and Subscription
+ */
+@Slf4j
+@Service
+public class KafkaResourceOperators implements QueueResourceOperator {
+
+ @Autowired
+ private InlongClusterService clusterService;
+ @Autowired
+ private InlongStreamService streamService;
+ @Autowired
+ private KafkaOperator kafkaOperator;
+ @Autowired
+ private ConsumptionService consumptionService;
+
+ @Override
+ public boolean accept(String mqType) {
+ return MQType.KAFKA.equals(mqType);
+ }
+
+ @Override
+ public void createQueueForGroup(@NotNull InlongGroupInfo groupInfo, @NotBlank String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+ Preconditions.checkNotNull(operator, "operator cannot be null");
+
+ String groupId = groupInfo.getInlongGroupId();
+ log.info("begin to create kafka resource for groupId={}", groupId);
+
+ // get kafka cluster via the inlong cluster tag from the inlong group
+ String clusterTag = groupInfo.getInlongClusterTag();
+ KafkaClusterInfo kafkaCluster = (KafkaClusterInfo) clusterService.getOne(clusterTag, null,
+ ClusterType.KAFKA);
+ try {
+ // 1. create kafka Topic - each Inlong Stream corresponds to a Kafka Topic
+ List<InlongStreamBriefInfo> streamInfoList = streamService.getTopicList(groupId);
+ if (streamInfoList == null || streamInfoList.isEmpty()) {
+ log.warn("skip to create kafka topic and subscription as no streams for groupId={}", groupId);
+ return;
+ }
+ for (InlongStreamBriefInfo streamInfo : streamInfoList) {
+ this.createKafkaTopic(groupInfo, kafkaCluster, streamInfo.getInlongStreamId());
+ }
+ } catch (Exception e) {
+ String msg = String.format("failed to create pulsar resource for groupId=%s", groupId);
Review Comment:
ditto
##########
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/kafka/KafkaResourceOperators.java:
##########
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.service.resource.queue.kafka;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.common.consts.MQType;
+import org.apache.inlong.manager.common.enums.ClusterType;
+import org.apache.inlong.manager.common.exceptions.WorkflowListenerException;
+import org.apache.inlong.manager.common.util.Preconditions;
+import org.apache.inlong.manager.pojo.cluster.ClusterInfo;
+import org.apache.inlong.manager.pojo.cluster.kafka.KafkaClusterInfo;
+import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamBriefInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.manager.service.cluster.InlongClusterService;
+import org.apache.inlong.manager.service.core.ConsumptionService;
+import org.apache.inlong.manager.service.resource.queue.QueueResourceOperator;
+import org.apache.inlong.manager.service.stream.InlongStreamService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * kafka for create kafka Topic and Subscription
+ */
+@Slf4j
+@Service
+public class KafkaResourceOperators implements QueueResourceOperator {
+
+ @Autowired
+ private InlongClusterService clusterService;
+ @Autowired
+ private InlongStreamService streamService;
+ @Autowired
+ private KafkaOperator kafkaOperator;
+ @Autowired
+ private ConsumptionService consumptionService;
+
+ @Override
+ public boolean accept(String mqType) {
+ return MQType.KAFKA.equals(mqType);
+ }
+
+ @Override
+ public void createQueueForGroup(@NotNull InlongGroupInfo groupInfo, @NotBlank String operator) {
+ Preconditions.checkNotNull(groupInfo, "inlong group info cannot be null");
+ Preconditions.checkNotNull(operator, "operator cannot be null");
Review Comment:
no need to call **_checkNotNull_** since the annotation @NotNull @NotBlank have ensure the avaliability.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org