[GitHub] [incubator-seatunnel] ic4y commented on a diff in pull request #2366: [ST-Engine][TaskExecutionService]Add dynamic thread sharing optimization

[GitBox <gi...@apache.org>]

ic4y commented on code in PR #2366:
URL: https://github.com/apache/incubator-seatunnel/pull/2366#discussion_r938557606


##########
seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java:
##########
@@ -156,7 +169,129 @@ private final class BlockingTaskThreadFactory implements ThreadFactory {
         @Override
         public Thread newThread(@NonNull Runnable r) {
             return new Thread(r,
-                String.format("hz.%s.seaTunnel.blocking.thread-%d", hzInstanceName, seq.getAndIncrement()));
+                String.format("hz.%s.seaTunnel.task.thread-%d", hzInstanceName, seq.getAndIncrement()));
+        }
+    }
+
+    /**
+     * BusWork is used to poll the task call method,
+     * When a task times out, a new BusWork will be created to take over the execution of the task
+     */
+
+    public final class BusWork implements Runnable {
+
+        AtomicBoolean keep = new AtomicBoolean(true);
+        public AtomicReference<TaskTracker> exclusiveTaskTracker = new AtomicReference<>();
+        final TaskCallTimer timer;
+        public LinkedBlockingDeque<TaskTracker> taskqueue;
+
+        @SuppressWarnings("checkstyle:MagicNumber")
+        public BusWork(LinkedBlockingDeque<TaskTracker> taskqueue, RunBusWorkSupplier runBusWorkSupplier) {
+            logger.info(String.format("Created new BusWork : %s", this.hashCode()));
+            this.taskqueue = taskqueue;
+            this.timer = new TaskCallTimer(50, keep, runBusWorkSupplier, this);
+        }
+
+        @SneakyThrows
+        @Override
+        public void run() {
+            while (keep.get()) {
+                TaskTracker taskTracker = null != exclusiveTaskTracker.get() ?
+                    exclusiveTaskTracker.get() :
+                    taskqueue.takeFirst();
+                NonCompletableFuture future = taskTracker.taskFuture;
+                if (taskTracker.taskRuntimeFutures.isCancelled()) {
+                    if (null != exclusiveTaskTracker.get()) {
+                        // If it's exclusive need to end the work
+                        break;
+                    } else {
+                        // No action required and don't put back
+                        continue;
+                    }
+                }
+                //start timer, if it's exclusive, don't need to start
+                if (null == exclusiveTaskTracker.get()) {
+                    timer.timerStart(taskTracker);
+                }
+                ProgressState call = null;
+                try {
+                    //run task
+                    call = taskTracker.task.call();
+                    synchronized (timer) {
+                        timer.timerStop();
+                    }
+                } catch (Throwable e) {
+                    //task Failure and complete
+                    future.internalCompleteExceptionally(e);
+                    //If it's exclusive need to end the work
+                    logger.warning("Exception in " + taskTracker.task, e);
+                    if (null != exclusiveTaskTracker.get()) {
+                        break;
+                    }
+                } finally {
+                    //stop timer
+                    timer.timerStop();
+                }
+                //task call finished
+                if (null != call) {
+                    if (call.isDone()) {
+                        //If it's exclusive, you need to end the work
+                        future.internalComplete();
+                        if (null != exclusiveTaskTracker.get()) {
+                            break;
+                        }
+                    } else {
+                        //Task is not completed. Put task to the end of the queue
+                        //If the current work has an exclusive tracker, it will not be put back
+                        if (null == exclusiveTaskTracker.get()) {
+                            taskqueue.offer(taskTracker);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Used to create a new BusWork and run
+     */
+    public final class RunBusWorkSupplier {
+
+        ExecutorService executorService;
+        LinkedBlockingDeque<TaskTracker> taskQueue;
+
+        public RunBusWorkSupplier(ExecutorService executorService, LinkedBlockingDeque<TaskTracker> taskqueue) {
+            this.executorService = executorService;
+            this.taskQueue = taskqueue;
+        }
+
+        public boolean runNewBusWork(boolean checkTaskQueue) {
+            if (!checkTaskQueue || taskQueue.size() > 0) {
+                executorService.submit(new BusWork(taskQueue, this));
+                return true;
+            }
+            return false;
+        }
+    }
+
+    /**
+     * The action to be performed when the task call method execution times out
+     */
+    private final class TimeoutAction implements Runnable {

Review Comment:
   Renamed to TimeoutAct



##########
seatunnel-engine/seatunnel-engine-server/src/main/test/execution/FixedCallTimeTask.java:
##########
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package execution;
+
+import org.apache.seatunnel.engine.server.execution.ProgressState;
+import org.apache.seatunnel.engine.server.execution.Task;
+
+import lombok.NonNull;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class FixedCallTimeTask implements Task {
+    long callTime;
+    String name;
+    long currentTime;
+    CopyOnWriteArrayList<Long> lagList;
+    AtomicBoolean stop;
+
+    public FixedCallTimeTask(long callTime, String name, AtomicBoolean stop, CopyOnWriteArrayList<Long> lagList){
+        this.callTime = callTime;
+        this.name = name;
+        this.stop = stop;
+        this.lagList = lagList;
+    }
+
+    @NonNull
+    @Override
+    public ProgressState call() {
+        if(currentTime != 0){
+            lagList.add(System.currentTimeMillis() - currentTime);
+        }
+        currentTime = System.currentTimeMillis();
+
+        try {
+            Thread.sleep(callTime);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e.toString());

Review Comment:
   This is required for testing,need to throw.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@seatunnel.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org