[incubator-inlong] branch master updated: [INLONG-3335][Agent] Fix agent snapshot mode won't work and optimize jvm parameters (#3336)

[zi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zirui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git


The following commit(s) were added to refs/heads/master by this push:
     new 1c35575  [INLONG-3335][Agent] Fix agent snapshot mode won't work and optimize jvm parameters (#3336)
1c35575 is described below

commit 1c355755764bb3a964e97913ce0a2427a2dcc69c
Author: Schnapps <zp...@connect.ust.hk>
AuthorDate: Thu Mar 24 12:23:35 2022 +0800

    [INLONG-3335][Agent] Fix agent snapshot mode won't work and optimize jvm parameters (#3336)
---
 .../apache/inlong/agent/plugin/sources/reader/BinlogReader.java  | 9 +++++----
 inlong-agent/bin/agent-env.sh                                    | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java
index d1748d0..dea2541 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java
@@ -72,13 +72,13 @@ public class BinlogReader implements Reader {
     private static final String JOB_DATABASE_SNAPSHOT_MODE = "job.binlogJob.snapshot.mode";
     private static final String JOB_DATABASE_HISTORY_MONITOR_DDL = "job.binlogJob.ddl";
     private static final String JOB_DATABASE_PORT = "job.binlogJob.port";
+    private static final String JOB_DATABASE_QUEUE_SIZE = "job.binlogJob.queueSize";
 
     /**
      * pair.left: table name
      * pair.right: actual data
      */
-    private LinkedBlockingQueue<Pair<String, String>> binlogMessagesQueue =
-        new LinkedBlockingQueue<>();
+    private LinkedBlockingQueue<Pair<String, String>> binlogMessagesQueue;
 
     private boolean finished = false;
     private String userName;
@@ -136,7 +136,7 @@ public class BinlogReader implements Reader {
         password = jobConf.get(JOB_DATABASE_PASSWORD);
         hostName = jobConf.get(JOB_DATABASE_HOSTNAME);
         port = jobConf.get(JOB_DATABASE_PORT);
-        tableWhiteList = jobConf.get(JOB_TABLE_WHITELIST, "");
+        tableWhiteList = jobConf.get(JOB_TABLE_WHITELIST, "[\\s\\S]*.*");
         databaseWhiteList = jobConf.get(JOB_DATABASE_WHITELIST, "");
         serverTimeZone = jobConf.get(JOB_DATABASE_SERVER_TIME_ZONE, "");
         offsetFlushIntervalMs = jobConf.get(JOB_DATABASE_STORE_OFFSET_INTERVAL_MS, "1000");
@@ -147,6 +147,7 @@ public class BinlogReader implements Reader {
         snapshotMode = jobConf.get(JOB_DATABASE_SNAPSHOT_MODE, "");
         includeSchemaChanges = jobConf.get(JOB_DATABASE_INCLUDE_SCHEMA_CHANGES, "false");
         historyMonitorDdl = jobConf.get(JOB_DATABASE_HISTORY_MONITOR_DDL, "false");
+        binlogMessagesQueue = new LinkedBlockingQueue<>(jobConf.getInt(JOB_DATABASE_QUEUE_SIZE, 10000));
         instanceId = jobConf.getInstanceId();
         finished = false;
 
@@ -185,7 +186,7 @@ public class BinlogReader implements Reader {
                     for (ChangeEvent<String, String> record : records) {
                         DebeziumFormat debeziumFormat = gson
                             .fromJson(record.value(), DebeziumFormat.class);
-                        binlogMessagesQueue.add(Pair.of(debeziumFormat.getSource().getTable(),
+                        binlogMessagesQueue.put(Pair.of(debeziumFormat.getSource().getTable(),
                             record.value()));
                         committer.markProcessed(record);
                     }
diff --git a/inlong-agent/bin/agent-env.sh b/inlong-agent/bin/agent-env.sh
index 523f408..4c70b95 100755
--- a/inlong-agent/bin/agent-env.sh
+++ b/inlong-agent/bin/agent-env.sh
@@ -36,7 +36,7 @@ else
   export JPS="$JAVA_HOME/bin/jps"
 fi
 
-HEAP_OPTS="-Xmx1024m -Xms512m"
+HEAP_OPTS="-Xms512m"
 GC_OPTS="-XX:SurvivorRatio=6 -XX:+UseMembar -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSScavengeBeforeRemark -XX:ParallelCMSThreads=3 -XX:+TieredCompilation -XX:+UseCMSCompactAtFullCollection -verbose:gc -Xloggc:$BASE_DIR/logs/gc.log.`date +%Y-%m-%d-%H-%M-%S` -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$BASE_DIR/logs/ -XX:+CMSClassUnloadingEnabled -XX:CMSInitiatingOccupancyFraction=60 -XX:CMSFullGCsBeforeCompaction=1 -Dsun [...]
 AGENT_JVM_ARGS="$HEAP_OPTS $GC_OPTS"