[incubator-linkis] 05/05: add kafka meta service

[pe...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

peacewong pushed a commit to branch dev-1.1.0-datasource
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit 9723a575a4daa58a23ee0e53b597fc51cb59162a
Author: xiaojie19852006 <xi...@163.com>
AuthorDate: Mon Feb 7 17:08:03 2022 +0800

    add kafka meta service
---
 .../metadatamanager/service/KafkaMetaService.java  | 111 +++++++++++++++++++++
 1 file changed, 111 insertions(+)

diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/kafka/src/main/java/org/apache/linkis/metadatamanager/service/KafkaMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/kafka/src/main/java/org/apache/linkis/metadatamanager/service/KafkaMetaService.java
new file mode 100644
index 0000000..70f9717
--- /dev/null
+++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/kafka/src/main/java/org/apache/linkis/metadatamanager/service/KafkaMetaService.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadatamanager.service;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.linkis.bml.client.BmlClient;
+import org.apache.linkis.bml.client.BmlClientFactory;
+import org.apache.linkis.bml.protocol.BmlDownloadResponse;
+import org.apache.linkis.common.conf.CommonVars;
+import org.apache.linkis.metadatamanager.common.exception.MetaRuntimeException;
+import org.apache.linkis.metadatamanager.common.service.AbstractMetaService;
+import org.apache.linkis.metadatamanager.common.service.MetadataConnection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+@Component
+public class KafkaMetaService extends AbstractMetaService<KafkaConnection> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(KafkaMetaService.class);
+    private static final CommonVars<String> TMP_FILE_STORE_LOCATION =
+            CommonVars.apply("wds.linkis.server.mdm.service.temp.location", "classpath:/tmp");
+    private BmlClient client;
+
+    @PostConstruct
+    public void buildClient(){
+        client = BmlClientFactory.createBmlClient();
+    }
+
+    @Override
+    public MetadataConnection<KafkaConnection> getConnection(String operator, Map<String, Object> params) throws Exception {
+        Resource resource = new PathMatchingResourcePatternResolver().getResource(TMP_FILE_STORE_LOCATION.getValue());
+        String brokers = String.valueOf(params.getOrDefault(KafkaParamsMapper.PARAM_KAFKA_BROKERS.getValue(), ""));
+        String principle = String.valueOf(params.getOrDefault(KafkaParamsMapper.PARAM_KAFKA_PRINCIPLE.getValue(), ""));
+        KafkaConnection conn;
+
+        if(StringUtils.isNotBlank(principle)){
+            LOG.info("Try to connect Kafka MetaStore in kerberos with principle:[" + principle +"]");
+            String keytabResourceId = String.valueOf(params.getOrDefault(KafkaParamsMapper.PARAM_KAFKA_KEYTAB.getValue(), ""));
+            if(StringUtils.isNotBlank(keytabResourceId)){
+                LOG.info("Start to download resource id:[" + keytabResourceId +"]");
+                String keytabFilePath = resource.getFile().getAbsolutePath() + "/" + UUID.randomUUID().toString().replace("-", "");
+                if(!downloadResource(keytabResourceId, operator, keytabFilePath)){
+                    throw new MetaRuntimeException("Fail to download resource i:[" + keytabResourceId +"]", null);
+                }
+                conn = new KafkaConnection(brokers, principle, keytabFilePath);
+            }else{
+                throw new MetaRuntimeException("Cannot find the keytab file in connect parameters", null);
+            }
+        }else{
+            conn = new KafkaConnection(brokers);
+        }
+
+        // because KafkaAdminClient.create does not do a real connection, we use listTopics here for testing connection
+        conn.getClient().listTopics().names().get();
+
+        return new MetadataConnection<>(conn, true);
+    }
+
+
+    @Override
+    public List<String> queryDatabases(KafkaConnection connection){
+        return Arrays.asList("default");
+    }
+
+    @Override
+    public List<String> queryTables(KafkaConnection connection, String database){
+        try {
+            return connection.getClient().listTopics().names().get().stream().collect(Collectors.toList());
+        } catch (Exception e) {
+            throw new RuntimeException("Fail to get Kafka topics(获取topic列表失败)", e);
+        }
+    }
+
+
+    private boolean downloadResource(String resourceId, String user, String absolutePath) throws IOException {
+        BmlDownloadResponse downloadResponse = client.downloadResource(user, resourceId, absolutePath);
+        if(downloadResponse.isSuccess()){
+            IOUtils.copy(downloadResponse.inputStream(), new FileOutputStream(absolutePath));
+            return true;
+        }
+        return false;
+    }
+}

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org