You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/06/18 02:34:01 UTC

[kylin] branch master updated (d05a23f -> 121f8d2)

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.


    from d05a23f  KYLIN-4343 Cleanup intermediate dict table
     new 6fc39b4  KYLIN-4577 Throws UnrecognizedPropertyException when server self discovery enables
     new 121f8d2  KYLIN-4499 Refine waiting time for initialization

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../common/zookeeper/KylinServerDiscovery.java     | 45 +++++++++++++++++++++-
 .../common/zookeeper/KylinServerDiscoveryTest.java |  4 +-
 .../kylin/job/impl/curator/CuratorScheduler.java   | 35 -----------------
 3 files changed, 45 insertions(+), 39 deletions(-)


[kylin] 01/02: KYLIN-4577 Throws UnrecognizedPropertyException when server self discovery enables

Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 6fc39b4d77493ce830555521c95766b5dca1513d
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Wed Jun 17 18:14:21 2020 +0800

    KYLIN-4577 Throws UnrecognizedPropertyException when server self discovery enables
    
    Problem:
    When server self discovery enables, it throws UnrecognizedPropertyException on some env, for example: cdh 5.7.
    
    Root cause:
    The json of ServiceInstance is incompatible, related to the issue of curator:  https://issues.apache.org/jira/browse/CURATOR-394
---
 .../common/zookeeper/KylinServerDiscovery.java     | 35 +++++++++++++++++++++-
 .../common/zookeeper/KylinServerDiscoveryTest.java |  4 +--
 .../kylin/job/impl/curator/CuratorScheduler.java   | 35 ----------------------
 3 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java b/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java
index 5b67d3e..07f59ad 100644
--- a/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java
+++ b/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.common.zookeeper;
 
+import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.LinkedHashMap;
@@ -33,7 +34,7 @@ import org.apache.curator.x.discovery.ServiceCache;
 import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.details.JsonInstanceSerializer;
+import org.apache.curator.x.discovery.details.InstanceSerializer;
 import org.apache.curator.x.discovery.details.ServiceCacheListener;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.StringUtil;
@@ -41,6 +42,9 @@ import org.apache.kylin.common.util.ZKUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -161,4 +165,33 @@ public class KylinServerDiscovery implements Closeable {
         IOUtils.closeQuietly(serviceCache);
         IOUtils.closeQuietly(serviceDiscovery);
     }
+
+    static class JsonInstanceSerializer<T> implements InstanceSerializer<T> {
+        private final ObjectMapper mapper;
+        private final Class<T> payloadClass;
+        private final JavaType type;
+
+        JsonInstanceSerializer(Class<T> payloadClass) {
+            this.payloadClass = payloadClass;
+            this.mapper = new ObjectMapper();
+
+            // to bypass https://issues.apache.org/jira/browse/CURATOR-394
+            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+            this.type = this.mapper.getTypeFactory().constructType(ServiceInstance.class);
+        }
+
+        public ServiceInstance<T> deserialize(byte[] bytes) throws Exception {
+            ServiceInstance rawServiceInstance = this.mapper.readValue(bytes, this.type);
+            this.payloadClass.cast(rawServiceInstance.getPayload());
+            return rawServiceInstance;
+        }
+
+        public byte[] serialize(ServiceInstance<T> instance) throws Exception {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            mapper.convertValue(instance.getPayload(), payloadClass);
+            this.mapper.writeValue(out, instance);
+            return out.toByteArray();
+        }
+    }
 }
diff --git a/core-common/src/test/java/org/apache/kylin/common/zookeeper/KylinServerDiscoveryTest.java b/core-common/src/test/java/org/apache/kylin/common/zookeeper/KylinServerDiscoveryTest.java
index efcb2ef..77dc673 100644
--- a/core-common/src/test/java/org/apache/kylin/common/zookeeper/KylinServerDiscoveryTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/zookeeper/KylinServerDiscoveryTest.java
@@ -29,7 +29,6 @@ import org.apache.curator.utils.CloseableUtils;
 import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.details.JsonInstanceSerializer;
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.common.util.ZKUtil;
 import org.junit.After;
@@ -77,7 +76,8 @@ public class KylinServerDiscoveryTest extends LocalFileMetadataTestCase {
         CuratorFramework curatorClient = null;
         try {
             String servicePath = KylinServerDiscovery.SERVICE_PATH;
-            final JsonInstanceSerializer<LinkedHashMap> serializer = new JsonInstanceSerializer<>(LinkedHashMap.class);
+            final KylinServerDiscovery.JsonInstanceSerializer<LinkedHashMap> serializer =
+                    new KylinServerDiscovery.JsonInstanceSerializer<>(LinkedHashMap.class);
             curatorClient = ZKUtil.newZookeeperClient(zkString, new ExponentialBackoffRetry(3000, 3));
             serviceDiscovery = ServiceDiscoveryBuilder.builder(LinkedHashMap.class).client(curatorClient)
                     .basePath(servicePath).serializer(serializer).build();
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/curator/CuratorScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/curator/CuratorScheduler.java
index 7e829ef..225de32 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/curator/CuratorScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/curator/CuratorScheduler.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.job.impl.curator;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.concurrent.Executors;
@@ -27,8 +26,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.details.InstanceSerializer;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.ServerMode;
 import org.apache.kylin.common.util.ZKUtil;
@@ -40,9 +37,6 @@ import org.apache.kylin.job.lock.JobLock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JavaType;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 
 public class CuratorScheduler implements Scheduler<AbstractExecutable> {
@@ -156,33 +150,4 @@ public class CuratorScheduler implements Scheduler<AbstractExecutable> {
         return jobClient;
     }
 
-    static class JsonInstanceSerializer<T> implements InstanceSerializer<T> {
-        private final ObjectMapper mapper;
-        private final Class<T> payloadClass;
-        private final JavaType type;
-
-        JsonInstanceSerializer(Class<T> payloadClass) {
-            this.payloadClass = payloadClass;
-            this.mapper = new ObjectMapper();
-
-            // to bypass https://issues.apache.org/jira/browse/CURATOR-394
-            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
-            this.type = this.mapper.getTypeFactory().constructType(ServiceInstance.class);
-        }
-
-        public ServiceInstance<T> deserialize(byte[] bytes) throws Exception {
-            ServiceInstance rawServiceInstance = this.mapper.readValue(bytes, this.type);
-            this.payloadClass.cast(rawServiceInstance.getPayload());
-            return rawServiceInstance;
-        }
-
-        public byte[] serialize(ServiceInstance<T> instance) throws Exception {
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
-            mapper.convertValue(instance.getPayload(), payloadClass);
-            this.mapper.writeValue(out, instance);
-            return out.toByteArray();
-        }
-    }
-
 }


[kylin] 02/02: KYLIN-4499 Refine waiting time for initialization

Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 121f8d23e3a918a6d93da716fe740bbcdaef56fc
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Wed Jun 17 19:30:35 2020 +0800

    KYLIN-4499 Refine waiting time for initialization
    
    yanghong's PR #1268
---
 .../apache/kylin/common/zookeeper/KylinServerDiscovery.java    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java b/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java
index 07f59ad..8c95670 100644
--- a/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java
+++ b/core-common/src/main/java/org/apache/kylin/common/zookeeper/KylinServerDiscovery.java
@@ -120,9 +120,17 @@ public class KylinServerDiscovery implements Closeable {
             serviceCache.start();
 
             registerSelf();
+            int i = 1;
+            long maxWaitingTime = 60 * 1000L; // 1 min
             while (!isFinishInit.get()) {
                 logger.info("Haven't registered, waiting ...");
-                Thread.sleep(100L);
+                long waitingTime = 100L * i * i;
+                if (waitingTime > maxWaitingTime) {
+                    waitingTime = maxWaitingTime;
+                } else {
+                    i++;
+                }
+                Thread.sleep(waitingTime);
             }
         } catch (Exception e) {
             throw new RuntimeException("Fail to initialize due to ", e);