You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2020/12/04 17:50:01 UTC
[iotdb] branch cluster_configuration_consensus created (now c228993)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a change to branch cluster_configuration_consensus
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at c228993 remove duplicated configuration files
This branch includes the following new commits:
new c228993 remove duplicated configuration files
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: remove duplicated configuration files
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch cluster_configuration_consensus
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c228993e9ecc430f5f82a1ba009640506086368f
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Sat Dec 5 01:48:33 2020 +0800
remove duplicated configuration files
---
cluster/src/assembly/cluster.xml | 4 +
.../src/assembly/resources/conf/cluster-env.bat | 81 -----
cluster/src/assembly/resources/conf/cluster-env.sh | 165 ---------
.../resources/conf/iotdb-engine.properties | 391 ---------------------
cluster/src/assembly/resources/conf/logback.xml | 253 -------------
.../apache/iotdb/cluster/server/NodeReport.java | 2 +-
docs/UserGuide/Server/Cluster Setup.md | 3 +
docs/UserGuide/Server/Config Manual.md | 29 ++
docs/zh/UserGuide/Server/Cluster Setup.md | 2 +
docs/zh/UserGuide/Server/Config Manual.md | 38 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../resources/conf/iotdb-engine.properties | 9 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 17 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +
.../db/integration/IoTDBRpcCompressionIT.java | 153 ++++++++
.../org/apache/iotdb/rpc/RpcTransportFactory.java | 10 +-
16 files changed, 264 insertions(+), 899 deletions(-)
diff --git a/cluster/src/assembly/cluster.xml b/cluster/src/assembly/cluster.xml
index 1cccdcb..5c0c581 100644
--- a/cluster/src/assembly/cluster.xml
+++ b/cluster/src/assembly/cluster.xml
@@ -19,5 +19,9 @@
<directory>src/assembly/resources</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>${maven.multiModuleProjectDirectory}/server/src/assembly/resources</directory>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
</fileSets>
</assembly>
diff --git a/cluster/src/assembly/resources/conf/cluster-env.bat b/cluster/src/assembly/resources/conf/cluster-env.bat
deleted file mode 100644
index 1bd28a5..0000000
--- a/cluster/src/assembly/resources/conf/cluster-env.bat
+++ /dev/null
@@ -1,81 +0,0 @@
-@REM
-@REM Licensed to the Apache Software Foundation (ASF) under one
-@REM or more contributor license agreements. See the NOTICE file
-@REM distributed with this work for additional information
-@REM regarding copyright ownership. The ASF licenses this file
-@REM to you under the Apache License, Version 2.0 (the
-@REM "License"); you may not use this file except in compliance
-@REM with the License. You may obtain a copy of the License at
-@REM
-@REM http://www.apache.org/licenses/LICENSE-2.0
-@REM
-@REM Unless required by applicable law or agreed to in writing,
-@REM software distributed under the License is distributed on an
-@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-@REM KIND, either express or implied. See the License for the
-@REM specific language governing permissions and limitations
-@REM under the License.
-@REM
-
-@echo off
-set LOCAL_JMX=no
-set JMX_PORT=31999
-
-if "%LOCAL_JMX%" == "yes" (
- set IOTDB_JMX_OPTS="-Diotdb.jmx.local.port=%JMX_PORT%" "-Dcom.sun.management.jmxremote.authenticate=false" "-Dcom.sun.management.jmxremote.ssl=false"
- ) else (
- set IOTDB_JMX_OPTS="-Dcom.sun.management.jmxremote" "-Dcom.sun.management.jmxremote.authenticate=false" "-Dcom.sun.management.jmxremote.ssl=false" "-Dcom.sun.management.jmxremote.port=%JMX_PORT%"
- )
-
-IF ["%IOTDB_HEAP_OPTS%"] EQU [""] (
- rem detect Java 8 or 11
- IF %%JAVA_VERSION%% == 8 (
- java -d64 -version >nul 2>&1
- IF NOT ERRORLEVEL 1 (
- rem 64-bit Java
- echo Detect 64-bit Java, maximum memory allocation pool = 2GB, initial memory allocation pool = 2GB
- set IOTDB_HEAP_OPTS=-Xmx4G -Xms4G -Xloggc:"%IOTDB_HOME%\gc.log" -XX:+PrintGCDateStamps
- -XX:+PrintGCDetails
- ) ELSE (
- rem 32-bit Java
- echo Detect 32-bit Java, maximum memory allocation pool = 512MB, initial memory allocation pool = 512MB
- set IOTDB_HEAP_OPTS=-Xmx512M -Xms512M -Xloggc:"%IOTDB_HOME%\gc.log" -XX:+PrintGCDateStamps -XX:+PrintGCDetails
- )
- goto end_config_setting
- ) ELSE (
- goto detect_jdk11_bit_version
- )
-)
-
-:detect_jdk11_bit_version
-for /f "tokens=1-3" %%j in ('java -version 2^>^&1') do (
- @rem echo %%j
- @rem echo %%k
- @rem echo %%l
- set BIT_VERSION=%%l
-)
-IF "%BIT_VERSION%" == "64-Bit" (
- rem 64-bit Java
- echo Detect 64-bit Java, maximum memory allocation pool = 2GB, initial memory allocation pool = 2GB
- set IOTDB_HEAP_OPTS=-Xmx4G -Xms4G
-) ELSE (
- rem 32-bit Java
- echo Detect 32-bit Java, maximum memory allocation pool = 512MB, initial memory allocation pool = 512MB
- set IOTDB_HEAP_OPTS=-Xmx512M -Xms512M
-)
-
-@REM IMPORTANT, avoid long gc which can stop the world, and cause timed out error.
-set IOTDB_HEAP_OPTS=%IOTDB_JMX_OPTS% -XX:MaxGCPauseMillis=5000
-
-:end_config_setting
-@REM set gc log.
-IF "%1" equ "printgc" (
- IF "%JAVA_VERSION%" == "8" (
- md %IOTDB_HOME%\logs
- set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -Xloggc:"%IOTDB_HOME%\logs\gc.log" -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
- ) ELSE (
- md %IOTDB_HOME%\logs
- set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file="%IOTDB_HOME%\logs\gc.log":time,uptime,pid,tid,level:filecount=10,filesize=10485760
- )
-)
-echo If you want to change this configuration, please check conf/iotdb-env.sh(Unix or OS X, if you use Windows, check conf/iotdb-env.bat).
\ No newline at end of file
diff --git a/cluster/src/assembly/resources/conf/cluster-env.sh b/cluster/src/assembly/resources/conf/cluster-env.sh
deleted file mode 100755
index facbb1d..0000000
--- a/cluster/src/assembly/resources/conf/cluster-env.sh
+++ /dev/null
@@ -1,165 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-calculate_heap_sizes()
-{
- case "`uname`" in
- Linux)
- system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
- system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo`
- ;;
- FreeBSD)
- system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
- system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
- system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
- ;;
- SunOS)
- system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
- system_cpu_cores=`psrinfo | wc -l`
- ;;
- Darwin)
- system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
- system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
- system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
- ;;
- *)
- # assume reasonable defaults for e.g. a modern desktop or
- # cheap server
- system_memory_in_mb="2048"
- system_cpu_cores="2"
- ;;
- esac
-
- # some systems like the raspberry pi don't report cores, use at least 1
- if [ "$system_cpu_cores" -lt "1" ]
- then
- system_cpu_cores="1"
- fi
-
- # set max heap size based on the following
- # max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
- # calculate 1/2 ram and cap to 1024MB
- # calculate 1/4 ram and cap to 8192MB
- # pick the max
- half_system_memory_in_mb=`expr $system_memory_in_mb / 2`
- quarter_system_memory_in_mb=`expr $half_system_memory_in_mb / 2`
- if [ "$half_system_memory_in_mb" -gt "1024" ]
- then
- half_system_memory_in_mb="1024"
- fi
- if [ "$quarter_system_memory_in_mb" -gt "8192" ]
- then
- quarter_system_memory_in_mb="8192"
- fi
- if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]
- then
- max_heap_size_in_mb="$half_system_memory_in_mb"
- else
- max_heap_size_in_mb="$quarter_system_memory_in_mb"
- fi
- MAX_HEAP_SIZE="${max_heap_size_in_mb}M"
-
- # Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 * heap size)
- max_sensible_yg_per_core_in_mb="100"
- max_sensible_yg_in_mb=`expr $max_sensible_yg_per_core_in_mb "*" $system_cpu_cores`
-
- desired_yg_in_mb=`expr $max_heap_size_in_mb / 4`
-
- if [ "$desired_yg_in_mb" -gt "$max_sensible_yg_in_mb" ]
- then
- HEAP_NEWSIZE="${max_sensible_yg_in_mb}M"
- else
- HEAP_NEWSIZE="${desired_yg_in_mb}M"
- fi
-}
-
-
-# Determine the sort of JVM we'll be running on.
-java_ver_output=`"${JAVA:-java}" -version 2>&1`
-jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
-JVM_VERSION=${jvmver%_*}
-JVM_PATCH_VERSION=${jvmver#*_}
-if [ "$JVM_VERSION" \< "1.8" ] ; then
- echo "IoTDB requires Java 8u40 or later."
- exit 1;
-fi
-
-if [ "$JVM_VERSION" \< "1.8" ] && [ "$JVM_PATCH_VERSION" -lt 40 ] ; then
- echo "IoTDB requires Java 8u40 or later."
- exit 1;
-fi
-
-version_arr=(${JVM_VERSION//./ })
-#GC log path has to be defined here because it needs to access IOTDB_HOME
-if [ "${version_arr[0]}" = "1" ] ; then
- # Java 8
- MAJOR_VERSION=${version_arr[1]}
- echo "$IOTDB_JMX_OPTS" | grep -q "^-[X]loggc"
- if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
- # only add -Xlog:gc if it's not mentioned in jvm-server.options file
- mkdir -p ${IOTDB_HOME}/logs
- if [ "$#" -ge "1" -a "$1" == "printgc" ]; then
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xloggc:${IOTDB_HOME}/logs/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M"
- fi
- fi
-else
- #JDK 11 and others
- MAJOR_VERSION=${version_arr[0]}
- # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
- # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
- echo "$IOTDB_JMX_OPTS" | grep -q "^-[X]log:gc"
- if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
- # only add -Xlog:gc if it's not mentioned in jvm-server.options file
- mkdir -p ${IOTDB_HOME}/logs
- if [ "$#" -ge "1" -a "$1" == "printgc" ]; then
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xlog:gc=info,heap*=info,age*=info,safepoint=info,promotion*=info:file=${IOTDB_HOME}/logs/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
- fi
- fi
-fi
-
-
-calculate_heap_sizes
-
-# Maximum heap size
-#MAX_HEAP_SIZE="2G"
-# Minimum heap size
-#HEAP_NEWSIZE="2G"
-
-JMX_LOCAL=no
-
-JMX_PORT="31999"
-
-if [ "JMX_LOCAL" = "yes" ]; then
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Diotdb.jmx.local.port=$JMX_PORT"
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
-else
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
-fi
-
-
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${HEAP_NEWSIZE}"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${MAX_HEAP_SIZE}"
-# IMPORTANT, avoid long gc which can stop the world, and cause timed out error.
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxGCPauseMillis=5000"
-
-echo "Maximum memory allocation pool = ${MAX_HEAP_SIZE}B, initial memory allocation pool = ${HEAP_NEWSIZE}B"
-echo "If you want to change this configuration, please check conf/iotdb-env.sh(Unix or OS X, if you use Windows, check conf/iotdb-env.bat)."
diff --git a/cluster/src/assembly/resources/conf/iotdb-engine.properties b/cluster/src/assembly/resources/conf/iotdb-engine.properties
deleted file mode 100644
index 19bc3e7..0000000
--- a/cluster/src/assembly/resources/conf/iotdb-engine.properties
+++ /dev/null
@@ -1,391 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-####################
-### Web Page Configuration
-####################
-
-metrics_port=8181
-
-####################
-### RPC Configuration
-####################
-
-rpc_address=0.0.0.0
-
-rpc_port=6667
-
-rpc_thrift_compression_enable=false
-
-rpc_max_concurrent_client_num=65535
-
-####################
-### Dynamic Parameter Adapter Configuration
-####################
-
-# Is dynamic parameter adapter enable. It's recommended for users to enable parameter adapter.
-# The adapter can dynamically adjust the following two parameters according to the memory load of the system:
-# 1. tsfile_size_threshold which is introduced below.
-# 2. memtable_size_threshold which is introduced below.
-# By dynamically adjusting these two parameters, the probability of system memory explosion is greatly reduced.
-# When this parameter is set true, it will refuse to create time series or add storage groups under high system load.
-enable_parameter_adapter=false
-
-####################
-### Write Ahead Log Configuration
-####################
-
-# Is insert ahead log enable
-enable_wal=true
-
-# When a certain amount of insert ahead log is reached, it will be flushed to disk
-# It is possible to lose at most flush_wal_threshold operations
-flush_wal_threshold=10000
-
-# The cycle when insert ahead log is periodically forced to be written to disk(in milliseconds)
-# If force_wal_period_in_ms = 0 it means force insert ahead log to be written to disk after each refreshment
-# Set this parameter to 0 may slow down the ingestion on slow disk.
-force_wal_period_in_ms=1000
-
-
-####################
-### Timestamp Precision Configuration
-####################
-# Use this value to set timestamp precision as "ms", "us" or "ns".
-# Once the precision is been set, it can not be changed.
-timestamp_precision=ms
-
-
-####################
-### Directory Configuration
-####################
-
-# base dir
-# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data).
-# If it is absolute, system will save the data in exact location it points to.
-# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
-# Note: If sys_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path.
-# For windows platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
-# base_dir=data
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# base_dir=data
-
-
-# data dirs
-# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/data).
-# If it is absolute, system will save the data in exact location it points to.
-# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
-# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path.
-# For windows platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
-# data_dirs=data\\data
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# data_dirs=data/data
-
-
-# mult_dir_strategy
-# The strategy is used to choose a directory from tsfile_dir for the system to store a new tsfile.
-# System provides three strategies to choose from, or user can create his own strategy by extending org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy.
-# The info of the three strategies are as follows:
-# 1. SequenceStrategy: the system will choose the directory in sequence.
-# 2. MaxDiskUsableSpaceFirstStrategy: the system will choose the directory whose disk has the maximum space.
-# 3. MinFolderOccupiedSpaceFirstStrategy: the system will choose the directory whose folder has the minimum occupied space.
-# Set SequenceStrategy,MaxDiskUsableSpaceFirstStrategy and MinFolderOccupiedSpaceFirstStrategy to apply the corresponding strategy.
-# If this property is unset, system will use MaxDiskUsableSpaceFirstStrategy as default strategy.
-# For this property, fully-qualified class name (include package name) and simple class name are both acceptable.
-# multi_dir_strategy=MaxDiskUsableSpaceFirstStrategy
-
-
-# wal dir
-# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data).
-# If it is absolute, system will save the data in the exact location it points to.
-# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
-# Note: If wal_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path.
-# For windows platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
-# wal_dir=data\\wal
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# wal_dir=data/wal
-
-
-# TSFile storage file system. Currently, Tsfile are supported to be stored in LOCAL file system or HDFS.
-tsfile_storage_fs=LOCAL
-
-# If using HDFS, hadoop ip can be configured
-hdfs_ip=localhost
-
-# If using HDFS, hadoop port can be configured
-hdfs_port=9000
-
-####################
-### Memory Control Configuration
-####################
-
-# Memory Allocation Ratio: Write, Read, and Free Memory.
-# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 6:3:1
-write_read_free_memory_proportion=6:3:1
-
-# The amount of data read each time in batch (the number of data strips, that is, the number of different timestamps.)
-fetch_size=10000
-
-# The two times the maximum buffer size of a single log(in byte).
-# If WAL is enabled and the size of an insert plan is smaller than half of this parameter,
-# then the insert plan will be rejected by WAL.
-# If it sets a value smaller than 0, use the default value 16777216
-wal_buffer_size=16777216
-
-# time zone of server side
-# default value is +08:00
-# eg. +08:00, -01:00
-time_zone=+08:00
-
-# When a TsFile's file size (in byte) exceeds this, the TsFile is forced closed. The default threshold is 512 MB.
-tsfile_size_threshold=0
-
-# When a memTable's size (in byte) exceeds this, the memtable is flushed to disk. The default threshold is 128 MB.
-memtable_size_threshold=134217728
-
-# How many threads can concurrently flush. When <= 0, use CPU core number.
-concurrent_flush_thread=0
-
-# whether take over the memory management by IoTDB rather than JVM when serializing memtable as bytes in memory
-# (i.e., whether use ChunkBufferPool), value true, false
-chunk_buffer_pool_enable=false
-
-# Default TTL for storage groups that are not set TTL by statements, in ms. If not set (default),
-# the TTL will be unlimited.
-# Notice: if this property is changed, previous created storage group which are not set TTL will
-# also be affected. And negative values are accepted, which means you can only insert future
-# data.
-# default_ttl=36000000
-
-####################
-### Merge Configurations
-####################
-
-enable_vm=false
-
-# How many thread will be set up to perform merge main tasks, 1 by default.
-# Set to 1 when less than or equal to 0.
-merge_thread_num=1
-
-# How many thread will be set up to perform merge chunk sub-tasks, 4 by default.
-# Set to 1 when less than or equal to 0.
-merge_chunk_subthread_num=4
-
-# If one merge file selection runs for more than this time, it will be ended and its current
-# selection will be used as final selection. Unit: millis.
-# When < 0, it means time is unbounded.
-merge_fileSelection_time_budget=30000
-
-# How much memory may be used in ONE merge task (in byte), 20% of maximum JVM memory by default.
-# This is only a rough estimation, starting from a relatively small value to avoid OOM.
-# Each new merge thread may take such memory, so merge_thread_num * merge_memory_budget is the
-# total memory estimation of merge.
-# merge_memory_budget=2147483648
-
-# When set to true, if some crashed merges are detected during system rebooting, such merges will
-# be continued, otherwise, the unfinished parts of such merges will not be continued while the
-# finished parts still remains as they are.
-# If you are feeling the rebooting is too slow, set this to false, false by default
-continue_merge_after_reboot=false
-
-# A global merge will be performed each such interval, that is, each storage group will be merged
-# (if proper merge candidates can be found). Unit: second, default: 1hours.
-# When less than or equal to 0, timed merge is disabled.
-merge_interval_sec=3600
-
-# When set to true, all merges becomes full merge (the whole SeqFiles are re-written despite how
-# much they are overflowed). This may increase merge overhead depending on how much the SeqFiles
-# are overflowed.
-force_full_merge=false
-
-# During a merge, if a chunk with less number of points than this parameter, the chunk will be
-# merged with its succeeding chunks even if it is not overflowed, until the merged chunks reach
-# this threshold and the new chunk will be flushed.
-# When less than 0, this mechanism is disabled.
-chunk_merge_point_threshold=20480
-
-####################
-### Metadata Cache Configuration
-####################
-
-# whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
-meta_data_cache_enable=true
-# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query.
-# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10
-filemeta_chunkmeta_free_memory_proportion=3:6:10
-
-# This cache is used to improve query speed where read from remote node
-remote_schema_cache_size=300000
-
-####################
-### Statistics Monitor configuration
-####################
-
-# Set enable_stat_monitor true(or false) to enable(or disable) the StatMonitor that stores statistics info periodically.
-# back_loop_period_sec decides the period when StatMonitor writes statistics info into IoTDB.
-# stat_monitor_detect_freq_sec decides when IoTDB detects statistics info out-of-date.
-# IoTDB just keeps statistics info within stat_monitor_retain_interval_sec seconds before current time.
-# Note: IoTDB requires stat_monitor_detect_freq_sec >= 600s and stat_monitor_retain_interval_sec >= 600s.
-# The monitor, which writes statistics info to IoTDB periodically, is disabled by default.
-enable_stat_monitor=false
-
-# The period that StatMonitor stores statistics info, the time unit is seconds.
-back_loop_period_in_second=5
-
-# The interval at which StatMonitor starts to check whether statistics info can be deleted due to exceeding the retention volume.
-# The time unit is seconds.
-stat_monitor_detect_freq_in_second=600
-
-# The minimum age of statistics storage information to be eligible for deletion due to age.
-# The time unit is seconds.
-stat_monitor_retain_interval_in_second=600
-
-# cache size for MManager.
-# This cache is used to improve insert speed where all path check and TSDataType will be cached in MManager with corresponding Path.
-schema_manager_cache_size=300000
-
-####################
-### External sort Configuration
-####################
-# Is external sort enable
-enable_external_sort=true
-
-# The maximum number of simultaneous chunk reading for a single time series.
-# If the num of simultaneous chunk reading is greater than external_sort_threshold, external sorting is used.
-# When external_sort_threshold increases, the number of chunks sorted at the same time in memory may increase and this will occupy more memory.
-# When external_sort_threshold decreases, triggering external sorting will increase the time-consuming.
-external_sort_threshold = 60
-
-
-####################
-### Sync Server Configuration
-####################
-
-# Whether to open the sync_server_port for receiving data from sync client, the default allowed
-is_sync_enable=false
-
-# Sync server port to listen
-sync_server_port=5555
-
-# White IP list of Sync client.
-# Please use the form of network segment to present the range of IP, for example: 192.168.0.0/16
-# If there are more than one IP segment, please separate them by commas
-# The default is to allow all IP to sync
-ip_white_list=0.0.0.0/0
-
-####################
-### performance statistic configuration
-####################
-
-# Is stat performance of sub-module enable
-enable_performance_stat=false
-# The interval of display statistic result in ms.
-performance_stat_display_interval=60000
-# The memory used for performance_stat in kb.
-performance_stat_memory_in_kb=20
-
-
-
-####################
-### Configurations for watermark module
-####################
-watermark_module_opened=false
-watermark_secret_key=IoTDB*2019@Beijing
-watermark_bit_string=100101110100
-watermark_method=GroupBasedLSBMethod(embed_row_cycle=2,embed_lsb_num=5)
-
-
-####################
-### Configurations for creating schema automatically
-####################
-
-# Whether creating schema automatically is enabled
-enable_auto_create_schema=true
-
-# Storage group level when creating schema automatically is enabled
-# e.g. root.sg0.d1.s2
-# we will set root.sg0 as the storage group if storage group level is 1
-default_storage_group_level=1
-
-# Boolean encoding when creating schema automatically is enabled
-default_boolean_encoding=RLE
-
-# Long encoding when creating schema automatically is enabled
-default_long_encoding=RLE
-
-# Double encoding when creating schema automatically is enabled
-default_double_encoding=GORILLA
-
-# String encoding when creating schema automatically is enabled
-default_string_encoding=PLAIN
-
-enable_mqtt_service=false
-
-# thrift max frame size
-thrift_max_frame_size=67108864
-
-# thrift init buffer size
-thrift_init_buffer_size=1024
-
-compaction_strategy=NO_COMPACTION
-
-####################
-### Memory Control Configuration
-####################
-
-# Whether to enable memory control
-enable_mem_control=true
-
-# Memory Allocation Ratio: Write, Read, Schema and Free Memory.
-# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 6:2:1:1
-write_read_schema_free_memory_proportion=7:1:1:2
-
-# primitive array size (length of each array) in array pool
-primitive_array_size=128
-
-# Ratio of write memory for invoking flush disk, 0.4 by default
-flush_proportion=0.4
-
-# Ratio of write memory allocated for buffered arrays, 0.6 by default
-buffered_arrays_memory_proportion=0.6
-
-# Ratio of write memory for rejecting insertion, 0.8 by default
-reject_proportion=0.85
-
-# If memory (in byte) of storage group increased more than this threshold, report to system. The default value is 16MB
-storage_group_report_threshold=16777216
-
-# allowed max numbers of deduplicated path in one query
-# it's just an advised value, the real limitation will be the smaller one between this and the one we calculated
-max_deduplicated_path_num=1000
-
-# When an inserting is rejected, waiting time (in ms) to check system again, 0 by default.
-waiting_time_when_insert_blocked=10
-
-# When the waiting time (in ms) of an inserting exceeds this, throw an exception. 0 by default.
-max_waiting_time_when_insert_blocked=10000
-
-# estimated metadata size (in byte) of one timeseries in Mtree
-estimated_series_size=300
\ No newline at end of file
diff --git a/cluster/src/assembly/resources/conf/logback.xml b/cluster/src/assembly/resources/conf/logback.xml
deleted file mode 100644
index e81e6e5..0000000
--- a/cluster/src/assembly/resources/conf/logback.xml
+++ /dev/null
@@ -1,253 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or a [...]
- -->
-<configuration scan="true" scanPeriod="60 seconds">
- <jmxConfigurator/>
- <!-- prevent logback from outputting its own status at the start of every log -->
- <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_ERROR">
- <file>${IOTDB_HOME}/logs/log_error.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-error-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>10MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>error</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_WARN">
- <file>${IOTDB_HOME}/logs/log_warn.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-warn-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>10MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>WARN</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_INFO">
- <file>${IOTDB_HOME}/logs/log_info.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-info-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>100MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>INFO</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_DEBUG">
- <file>${IOTDB_HOME}/logs/log_debug.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-debug-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>100MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>DEBUG</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_TRACE">
- <file>${IOTDB_HOME}/logs/log_trace.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-trace-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>100MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>TRACE</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.ConsoleAppender" name="FILE_STDOUT">
- <Target>System.out</Target>
- <encoder>
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- </appender>
- <logger level="info" name="org.apache.iotdb.db.service"/>
- <logger level="info" name="org.apache.iotdb.db.conf"/>
- <!-- a log appender that collect all log records whose level is greather than debug-->
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_ALL">
- <file>${IOTDB_HOME}/logs/log_all.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-all-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>100MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>DEBUG</level>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_COST_MEASURE">
- <file>${IOTDB_HOME}/logs/log_measure.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log-measure-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>50MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- </appender>
- <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE_DYNAMIC_PARAMETER">
- <file>${IOTDB_HOME}/logs/log_dynamic_adapter.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${IOTDB_HOME}/logs/log_dynamic_adapter-%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>10</maxIndex>
- </rollingPolicy>
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>50MB</maxFileSize>
- </triggeringPolicy>
- <append>true</append>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
- <charset>utf-8</charset>
- </encoder>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>DEBUG</level>
- </filter>
- </appender>
- <appender name="ASYNC_FILE_ERROR" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_ERROR"/>
- </appender>
- <appender name="ASYNC_FILE_WARN" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_WARN"/>
- </appender>
- <appender name="ASYNC_FILE_INFO" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_INFO"/>
- </appender>
- <appender name="ASYNC_FILE_DEBUG" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_DEBUG"/>
- </appender>
- <appender name="ASYNC_FILE_TRACE" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_TRACE"/>
- </appender>
- <appender name="ASYNC_FILE_ALL" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_ALL"/>
- </appender>
- <appender name="ASYNC_FILE_COST_MEASURE" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_COST_MEASURE"/>
- </appender>
- <appender name="ASYNC_FILE_DYNAMIC_PARAMETER" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_DYNAMIC_PARAMETER"/>
- </appender>
- <appender name="ASYNC_FILE_STDOUT" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>1024</queueSize>
- <neverBlock>true</neverBlock>
- <includeCallerData>true</includeCallerData>
- <appender-ref ref="FILE_STDOUT"/>
- </appender>
- <root level="info">
- <appender-ref ref="ASYNC_FILE_TRACE"/>
- <appender-ref ref="ASYNC_FILE_DEBUG"/>
- <appender-ref ref="ASYNC_FILE_INFO"/>
- <appender-ref ref="ASYNC_FILE_WARN"/>
- <appender-ref ref="ASYNC_FILE_ERROR"/>
- <appender-ref ref="ASYNC_FILE_ALL"/>
- <appender-ref ref="ASYNC_FILE_STDOUT"/>
- </root>
- <logger level="info" name="org.apache.iotdb.db.cost.statistic">
- <appender-ref ref="FILE_COST_MEASURE"/>
- </logger>
- <logger level="info" name="org.apache.iotdb.cluster.server.heartbeat"/>
- <logger level="error" name="org.apache.thrift.server.AbstractNonblockingServer"/>
- <logger level="warn" name="org.apache.thrift.server.TThreadPoolServer"/>
-</configuration>
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/NodeReport.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/NodeReport.java
index 0b6816c..e175248 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/NodeReport.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/NodeReport.java
@@ -118,7 +118,7 @@ public class NodeReport {
= (double) readBytes / readCompressedBytes;
double writeCompressionRatio = (double) writeBytes / writeCompressedBytes;
String transportCompressionReport = "";
- if (RpcTransportFactory.USE_SNAPPY) {
+ if (RpcTransportFactory.isUseSnappy()) {
transportCompressionReport =
", readBytes=" + readBytes + "/" + readCompressedBytes + "(" + readCompressionRatio +
")" +
diff --git a/docs/UserGuide/Server/Cluster Setup.md b/docs/UserGuide/Server/Cluster Setup.md
index 335716d..bbb00e9 100644
--- a/docs/UserGuide/Server/Cluster Setup.md
+++ b/docs/UserGuide/Server/Cluster Setup.md
@@ -19,6 +19,9 @@
-->
+__NOTICE: CURRENT IoTDB CLUSTER IS FOR TESTING NOW!
+PLEASE BE DELIBERATE IF YOU RUN IT IN PRODUCT ENVIRONMENT.__
+
# Cluster Setup
For installation prerequisites, please refer to [Installation Prerequisites](../Get%20Started/QuickStart.md)
## Prerequisite
diff --git a/docs/UserGuide/Server/Config Manual.md b/docs/UserGuide/Server/Config Manual.md
index 6ac7e34..647b8c2 100644
--- a/docs/UserGuide/Server/Config Manual.md
+++ b/docs/UserGuide/Server/Config Manual.md
@@ -227,6 +227,25 @@ The permission definitions are in ${IOTDB\_CONF}/conf/jmx.access.
|Default| 6667 |
|Effective|After restart system|
+* rpc\_thrift\_compression\_enable
+
+|Name| rpc\_thrift\_compression\_enable |
+|:---:|:---|
+|Description| Whether enable thrift's compression (using GZIP).|
+|Type|Boolean|
+|Default| false |
+|Effective|After restart system|
+
+* rpc\_advanced\_compression\_enable
+
+|Name| rpc\_advanced\_compression\_enable |
+|:---:|:---|
+|Description| Whether enable thrift's advanced compression.|
+|Type|Boolean|
+|Default| false |
+|Effective|After restart system|
+
+
* time\_zone
|Name| time\_zone |
@@ -564,6 +583,16 @@ The permission definitions are in ${IOTDB\_CONF}/conf/jmx.access.
|Default|no |
|Effective|After restart system|
+* thrift\_max\_frame\_size
+
+|Name| thrift\_max\_frame\_size |
+|:---:|:---|
+|Description| the max bytes in a RPC request/response|
+|Type| long |
+|Default| 67108864 (should >= 8 * 1024 * 1024) |
+|Effective|After restart system|
+
+
## Automatic Schema Creation and Type Inference
* enable\_auto\_create\_schema
diff --git a/docs/zh/UserGuide/Server/Cluster Setup.md b/docs/zh/UserGuide/Server/Cluster Setup.md
index 0330e40..4738eed 100644
--- a/docs/zh/UserGuide/Server/Cluster Setup.md
+++ b/docs/zh/UserGuide/Server/Cluster Setup.md
@@ -19,6 +19,8 @@
-->
+__集群模式目前是测试版！请谨慎在生产环境中使用。__
+
# 集群设置
安装环境请参考[安装环境](../Get%20Started/QuickStart.md)
## 前提条件
diff --git a/docs/zh/UserGuide/Server/Config Manual.md b/docs/zh/UserGuide/Server/Config Manual.md
index 780dc1b..57af604 100644
--- a/docs/zh/UserGuide/Server/Config Manual.md
+++ b/docs/zh/UserGuide/Server/Config Manual.md
@@ -343,23 +343,41 @@
|默认值| MaxDiskUsableSpaceFirstStrategy |
|改后生效方式|触发生效|
-* rpc_address
+* rpc\_address
-|名字| rpc_address |
+|名字| rpc\_address |
|:---:|:---|
|描述| |
|类型|String|
|默认值| "0.0.0.0" |
|改后生效方式|重启服务器生效|
-* rpc_port
+* rpc\_port
-|名字| rpc_port |
+|名字| rpc\_port |
|:---:|:---|
|描述|jdbc服务监听端口。请确认该端口不是系统保留端口并且未被占用。|
|类型|Short Int : [0,65535]|
|默认值| 6667 |
-|改后生效方式|重启服务器生效||
+|改后生效方式|重启服务器生效|
+
+* rpc\_thrift\_compression\_enable
+
+|名字| rpc\_thrift\_compression\_enable |
+|:---:|:---|
+|描述|是否启用thrift的压缩机制。|
+|类型|true 或者 false|
+|默认值| false |
+|改后生效方式|重启服务器生效|
+
+* rpc\_advanced\_compression\_enable
+
+|名字| rpc\_advanced\_compression\_enable |
+|:---:|:---|
+|描述|是否启用thrift的自定制压缩机制。|
+|类型|true 或者 false|
+|默认值| false |
+|改后生效方式|重启服务器生效|
* time\_zone
@@ -517,6 +535,16 @@
|改后生效方式|重启服务器生效|
+* thrift\_max\_frame\_size
+
+|名字| thrift\_max\_frame\_size |
+|:---:|:---|
+|描述| RPC 请求/响应的最大字节数|
+|类型| long |
+|默认值| 67108864 (应大于等于 8 * 1024 * 1024) |
+|改后生效方式|重启服务器生效|
+
+
## 数据类型自动推断
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
index 863a5ea..da5bae6 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
@@ -41,7 +41,7 @@ public class TsFileSequenceRead {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public static void main(String[] args) throws IOException {
- String filename = "";
+ String filename = "test.tsfile";
if (args.length >= 1) {
filename = args[0];
}
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 5c7851d..86757b6 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -37,8 +37,17 @@ rpc_port=6667
rpc_thrift_compression_enable=false
+# if true, a snappy based compression method will be called before sending data by the network
+rpc_advanced_compression_enable=false
+
rpc_max_concurrent_client_num=65535
+# thrift max frame size, 64MB by default
+thrift_max_frame_size=67108864
+
+# thrift init buffer size
+thrift_init_buffer_size=1024
+
####################
### Write Ahead Log Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index aebe4a8..b620ab9 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
import org.apache.iotdb.db.exception.LoadConfigurationException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.service.TSServiceImpl;
+import org.apache.iotdb.rpc.RpcTransportFactory;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -110,6 +111,11 @@ public class IoTDBConfig {
private boolean rpcThriftCompressionEnable = false;
/**
+ * whether to use Snappy compression before sending data through the network
+ */
+ private boolean rpcAdvancedCompressionEnable = false;
+
+ /**
* Port which the JDBC server listens to.
*/
private int rpcPort = 6667;
@@ -1516,7 +1522,7 @@ public class IoTDBConfig {
return rpcThriftCompressionEnable;
}
- void setRpcThriftCompressionEnable(boolean rpcThriftCompressionEnable) {
+ public void setRpcThriftCompressionEnable(boolean rpcThriftCompressionEnable) {
this.rpcThriftCompressionEnable = rpcThriftCompressionEnable;
}
@@ -2093,4 +2099,13 @@ public class IoTDBConfig {
public void setDefaultIndexWindowRange(int defaultIndexWindowRange) {
this.defaultIndexWindowRange = defaultIndexWindowRange;
}
+
+ public boolean isRpcAdvancedCompressionEnable() {
+ return rpcAdvancedCompressionEnable;
+ }
+
+ public void setRpcAdvancedCompressionEnable(boolean rpcAdvancedCompressionEnable) {
+ this.rpcAdvancedCompressionEnable = rpcAdvancedCompressionEnable;
+ RpcTransportFactory.setUseSnappy(this.rpcAdvancedCompressionEnable);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 16fad25..2a4827e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -173,6 +173,10 @@ public class IoTDBDescriptor {
Boolean.parseBoolean(properties.getProperty("rpc_thrift_compression_enable",
Boolean.toString(conf.isRpcThriftCompressionEnable()))));
+ conf.setRpcAdvancedCompressionEnable(
+ Boolean.parseBoolean(properties.getProperty("rpc_advanced_compression_enable",
+ Boolean.toString(conf.isRpcAdvancedCompressionEnable()))));
+
conf.setRpcPort(Integer.parseInt(properties.getProperty("rpc_port",
Integer.toString(conf.getRpcPort()))));
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRpcCompressionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRpcCompressionIT.java
new file mode 100644
index 0000000..615d35f
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRpcCompressionIT.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration;
+
+import static org.junit.Assert.assertEquals;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.rpc.RpcTransportFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class IoTDBRpcCompressionIT {
+ boolean rpcThriftCompression = IoTDBDescriptor.getInstance().getConfig().isRpcThriftCompressionEnable();
+ boolean rpcAdvancedCompression = IoTDBDescriptor.getInstance().getConfig().isRpcAdvancedCompressionEnable();
+ @Before
+ public void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig().setRpcThriftCompressionEnable(rpcThriftCompression);
+ IoTDBDescriptor.getInstance().getConfig().setRpcAdvancedCompressionEnable(rpcAdvancedCompression);
+ }
+
+ @Test
+ public void testNoRpcCompression()
+ throws SQLException, InterruptedException {
+ EnvironmentUtils.envSetUp();
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()
+ ){
+ doSomething(statement);
+ }
+ }
+
+ @Test
+ public void testRpcThriftCompression()
+ throws SQLException, InterruptedException {
+ IoTDBDescriptor.getInstance().getConfig().setRpcThriftCompressionEnable(true);
+ EnvironmentUtils.envSetUp();
+ //let JDBC use thrift rpc compression
+ Config.rpcThriftCompressionEnable = true;
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()
+ ){
+ doSomething(statement);
+ } finally {
+ Config.rpcThriftCompressionEnable = false;
+ }
+ }
+
+ @Test
+ public void testRpcAdvancedCompression()
+ throws SQLException, InterruptedException {
+ IoTDBDescriptor.getInstance().getConfig().setRpcAdvancedCompressionEnable(true);
+ EnvironmentUtils.envSetUp();
+ //let JDBC use thrift advanced compression
+ RpcTransportFactory.setUseSnappy(true);
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()
+ ){
+ doSomething(statement);
+ }
+ }
+
+ @Test
+ public void testBothRpcCompression()
+ throws SQLException, InterruptedException {
+ IoTDBDescriptor.getInstance().getConfig().setRpcThriftCompressionEnable(true);
+ IoTDBDescriptor.getInstance().getConfig().setRpcThriftCompressionEnable(true);
+ EnvironmentUtils.envSetUp();
+ //let JDBC use thrift rpc compression
+ Config.rpcThriftCompressionEnable = true;
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
+ "root");
+ Statement statement = connection.createStatement()
+ ){
+ doSomething(statement);
+ } finally {
+ Config.rpcThriftCompressionEnable = false;
+ }
+ }
+
+ private void doSomething(Statement statement) throws SQLException, InterruptedException {
+
+ statement.execute("set storage group to root.demo");
+ statement.execute("create timeseries root.demo.d1.s1 with datatype=INT64,encoding=RLE");
+ statement.execute("create timeseries root.demo.d1.s2 with datatype=INT64,encoding=RLE");
+ statement.execute("create timeseries root.demo.d1.s3 with datatype=INT64,encoding=RLE");
+ statement.execute("insert into root.demo.d1(time,s1,s2) values(1,1,2)");
+ statement.execute("flush");
+ statement.execute("insert into root.demo.d1(time,s3) values(1,1)");
+ statement.execute("flush");
+ try (ResultSet set = statement.executeQuery("SELECT * FROM root")) {
+ int cnt = 0;
+ while (set.next()) {
+ cnt++;
+ assertEquals(1, set.getLong("root.demo.d1.s3"));
+ }
+ assertEquals(1, cnt);
+ }
+ statement.execute("merge");
+ Thread.sleep(1000);
+ // before merge completes
+ try (ResultSet set = statement.executeQuery("SELECT * FROM root")) {
+ int cnt = 0;
+ while (set.next()) {
+ cnt++;
+ assertEquals(1, set.getLong("root.demo.d1.s3"));
+ }
+ assertEquals(1, cnt);
+ }
+
+ // after merge completes
+ statement.execute("DELETE FROM root.demo.d1");
+ }
+}
diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java
index 9121f48..57907de 100644
--- a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java
+++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java
@@ -27,7 +27,7 @@ import org.apache.thrift.transport.TTransportFactory;
public class RpcTransportFactory extends TTransportFactory {
// TODO: make it a config
- public static final boolean USE_SNAPPY = false;
+ public static boolean USE_SNAPPY = false;
public static final RpcTransportFactory INSTANCE;
static {
INSTANCE = USE_SNAPPY ?
@@ -45,4 +45,12 @@ public class RpcTransportFactory extends TTransportFactory {
public TTransport getTransport(TTransport trans) {
return inner.getTransport(trans);
}
+
+ public static boolean isUseSnappy() {
+ return USE_SNAPPY;
+ }
+
+ public static void setUseSnappy(boolean useSnappy) {
+ USE_SNAPPY = useSnappy;
+ }
}