You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2022/10/05 11:00:53 UTC
[iotdb] branch research/M4-visualization updated: decode 80%
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by this push:
new cd64028c3a decode 80%
cd64028c3a is described below
commit cd64028c3a4e0c3820b71e5f6f49d5bcd54469ea
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Oct 5 19:01:05 2022 +0800
decode 80%
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 483 ++++++---------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 +-
.../iotdb/session/MyRealDataWriteQueryTest.java | 125 +++---
.../iotdb/tsfile/common/conf/TSFileConfig.java | 5 +-
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 28 +-
.../encoding/decoder/DeltaBinaryDecoder.java | 71 ++-
.../encoding/encoder/DeltaBinaryEncoder.java | 29 +-
8 files changed, 281 insertions(+), 509 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index e275639edd..725c38d571 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -70,122 +70,76 @@ public class IoTDBConfig {
public static final Pattern NODE_PATTERN = Pattern.compile(NODE_MATCHER);
- /**
- * Port which the metrics service listens to.
- */
+ /** Port which the metrics service listens to. */
private int metricsPort = 8181;
private boolean enableMetricService = false;
- /**
- * whether to enable the mqtt service.
- */
+ /** whether to enable the mqtt service. */
private boolean enableMQTTService = false;
- /**
- * the mqtt service binding host.
- */
+ /** the mqtt service binding host. */
private String mqttHost = "0.0.0.0";
- /**
- * the mqtt service binding port.
- */
+ /** the mqtt service binding port. */
private int mqttPort = 1883;
- /**
- * the handler pool size for handing the mqtt messages.
- */
+ /** the handler pool size for handing the mqtt messages. */
private int mqttHandlerPoolSize = 1;
- /**
- * the mqtt message payload formatter.
- */
+ /** the mqtt message payload formatter. */
private String mqttPayloadFormatter = "json";
- /**
- * max mqtt message size
- */
+ /** max mqtt message size */
private int mqttMaxMessageSize = 1048576;
- /**
- * Rpc binding address.
- */
+ /** Rpc binding address. */
private String rpcAddress = "0.0.0.0";
- /**
- * whether to use thrift compression.
- */
+ /** whether to use thrift compression. */
private boolean rpcThriftCompressionEnable = false;
- /**
- * whether to use Snappy compression before sending data through the network
- */
+ /** whether to use Snappy compression before sending data through the network */
private boolean rpcAdvancedCompressionEnable = false;
- /**
- * Port which the JDBC server listens to.
- */
+ /** Port which the JDBC server listens to. */
private int rpcPort = 6667;
- /**
- * Max concurrent client number
- */
+ /** Max concurrent client number */
private int rpcMaxConcurrentClientNum = 65535;
- /**
- * Memory allocated for the write process
- */
+ /** Memory allocated for the write process */
private long allocateMemoryForWrite = Runtime.getRuntime().maxMemory() * 4 / 10;
- /**
- * Memory allocated for the read process
- */
+ /** Memory allocated for the read process */
private long allocateMemoryForRead = Runtime.getRuntime().maxMemory() * 3 / 10;
- /**
- * Memory allocated for the mtree
- */
+ /** Memory allocated for the mtree */
private long allocateMemoryForSchema = Runtime.getRuntime().maxMemory() * 1 / 10;
- /**
- * Memory allocated for the read process besides cache
- */
+ /** Memory allocated for the read process besides cache */
private long allocateMemoryForReadWithoutCache = allocateMemoryForRead * 3 / 10;
private volatile int maxQueryDeduplicatedPathNum = 1000;
- /**
- * Ratio of memory allocated for buffered arrays
- */
+ /** Ratio of memory allocated for buffered arrays */
private double bufferedArraysMemoryProportion = 0.6;
- /**
- * Flush proportion for system
- */
+ /** Flush proportion for system */
private double flushProportion = 0.4;
- /**
- * Reject proportion for system
- */
+ /** Reject proportion for system */
private double rejectProportion = 0.8;
- /**
- * If storage group increased more than this threshold, report to system.
- */
+ /** If storage group increased more than this threshold, report to system. */
private long storageGroupSizeReportThreshold = 16 * 1024 * 1024L;
- /**
- * When inserting rejected, waiting period to check system again
- */
+ /** When inserting rejected, waiting period to check system again */
private int checkPeriodWhenInsertBlocked = 50;
- /**
- * When inserting rejected exceeds this, throw an exception
- */
+ /** When inserting rejected exceeds this, throw an exception */
private int maxWaitingTimeWhenInsertBlockedInMs = 10000;
- /**
- * Is the write ahead log enable.
- */
+ /** Is the write ahead log enable. */
private boolean enableWal = true;
private volatile boolean readOnly = false;
@@ -198,9 +152,7 @@ public class IoTDBConfig {
*/
private int flushWalThreshold = 10000;
- /**
- * this variable set timestamp precision as millisecond, microsecond or nanosecond
- */
+ /** this variable set timestamp precision as millisecond, microsecond or nanosecond */
private String timestampPrecision = "ms";
/**
@@ -228,19 +180,13 @@ public class IoTDBConfig {
*/
private int mlogBufferSize = 1024 * 1024;
- /**
- * default base dir, stores all IoTDB runtime files
- */
+ /** default base dir, stores all IoTDB runtime files */
private static final String DEFAULT_BASE_DIR = "data";
- /**
- * System directory, including version file for each storage group and metadata
- */
+ /** System directory, including version file for each storage group and metadata */
private String systemDir = DEFAULT_BASE_DIR + File.separator + IoTDBConstant.SYSTEM_FOLDER_NAME;
- /**
- * Schema directory, including storage set of values.
- */
+ /** Schema directory, including storage set of values. */
private String schemaDir =
DEFAULT_BASE_DIR
+ File.separator
@@ -248,9 +194,7 @@ public class IoTDBConfig {
+ File.separator
+ IoTDBConstant.SCHEMA_FOLDER_NAME;
- /**
- * Sync directory, including the lock file, uuid file, device owner map
- */
+ /** Sync directory, including the lock file, uuid file, device owner map */
private String syncDir =
DEFAULT_BASE_DIR
+ File.separator
@@ -258,75 +202,47 @@ public class IoTDBConfig {
+ File.separator
+ IoTDBConstant.SYNC_FOLDER_NAME;
- /**
- * Performance tracing directory, stores performance tracing files
- */
+ /** Performance tracing directory, stores performance tracing files */
private String tracingDir = DEFAULT_BASE_DIR + File.separator + IoTDBConstant.TRACING_FOLDER_NAME;
- /**
- * Query directory, stores temporary files of query
- */
+ /** Query directory, stores temporary files of query */
private String queryDir = DEFAULT_BASE_DIR + File.separator + IoTDBConstant.QUERY_FOLDER_NAME;
- /**
- * External lib directory, stores user-uploaded JAR files
- */
+ /** External lib directory, stores user-uploaded JAR files */
private String extDir = IoTDBConstant.EXT_FOLDER_NAME;
- /**
- * External lib directory for UDF, stores user-uploaded JAR files
- */
+ /** External lib directory for UDF, stores user-uploaded JAR files */
private String udfDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.UDF_FOLDER_NAME;
- /**
- * Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"};
- */
+ /** Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"}; */
private String[] dataDirs = {"data" + File.separator + "data"};
- /**
- * Strategy of multiple directories.
- */
+ /** Strategy of multiple directories. */
private String multiDirStrategyClassName = null;
- /**
- * Wal directory.
- */
+ /** Wal directory. */
private String walDir = DEFAULT_BASE_DIR + File.separator + "wal";
- /**
- * Maximum MemTable number. Invalid when enableMemControl is true.
- */
+ /** Maximum MemTable number. Invalid when enableMemControl is true. */
private int maxMemtableNumber = 0;
- /**
- * The amount of data iterate each time in server
- */
+ /** The amount of data iterate each time in server */
private int batchSize = 100000;
- /**
- * How many threads can concurrently flush. When <= 0, use CPU core number.
- */
+ /** How many threads can concurrently flush. When <= 0, use CPU core number. */
private int concurrentFlushThread = Runtime.getRuntime().availableProcessors();
- /**
- * How many threads can concurrently query. When <= 0, use CPU core number.
- */
+ /** How many threads can concurrently query. When <= 0, use CPU core number. */
private int concurrentQueryThread = 8;
- /**
- * Is the write mem control for writing enable.
- */
+ /** Is the write mem control for writing enable. */
private boolean enableMemControl = true;
- /**
- * Is the write ahead log enable.
- */
+ /** Is the write ahead log enable. */
private boolean enableIndex = false;
- /**
- * How many threads can concurrently build index. When <= 0, use CPU core number.
- */
+ /** How many threads can concurrently build index. When <= 0, use CPU core number. */
private int concurrentIndexBuildThread = Runtime.getRuntime().availableProcessors();
/**
@@ -346,29 +262,19 @@ public class IoTDBConfig {
*/
private int defaultIndexWindowRange = 10;
- /**
- * index directory.
- */
+ /** index directory. */
private String indexRootFolder = "data" + File.separator + "index";
- /**
- * When a unSequence TsFile's file size (in byte) exceed this, the TsFile is forced closed.
- */
+ /** When a unSequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
private long unSeqTsFileSize = 1L;
- /**
- * When a sequence TsFile's file size (in byte) exceed this, the TsFile is forced closed.
- */
+ /** When a sequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
private long seqTsFileSize = 1L;
- /**
- * When a memTable's size (in byte) exceeds this, the memtable is flushed to disk.
- */
+ /** When a memTable's size (in byte) exceeds this, the memtable is flushed to disk. */
private long memtableSizeThreshold = 1024 * 1024 * 1024L;
- /**
- * Whether to timed flush sequence tsfiles' memtables.
- */
+ /** Whether to timed flush sequence tsfiles' memtables. */
private boolean enableTimedFlushSeqMemtable = false;
/**
@@ -377,14 +283,10 @@ public class IoTDBConfig {
*/
private long seqMemtableFlushInterval = 60 * 60 * 1000L;
- /**
- * The interval to check whether sequence memtables need flushing. Unit: ms
- */
+ /** The interval to check whether sequence memtables need flushing. Unit: ms */
private long seqMemtableFlushCheckInterval = 10 * 60 * 1000L;
- /**
- * Whether to timed flush unsequence tsfiles' memtables.
- */
+ /** Whether to timed flush unsequence tsfiles' memtables. */
private boolean enableTimedFlushUnseqMemtable = true;
/**
@@ -393,14 +295,10 @@ public class IoTDBConfig {
*/
private long unseqMemtableFlushInterval = 60 * 60 * 1000L;
- /**
- * The interval to check whether unsequence memtables need flushing. Unit: ms
- */
+ /** The interval to check whether unsequence memtables need flushing. Unit: ms */
private long unseqMemtableFlushCheckInterval = 10 * 60 * 1000L;
- /**
- * Whether to timed close tsfiles.
- */
+ /** Whether to timed close tsfiles. */
private boolean enableTimedCloseTsFile = true;
/**
@@ -409,14 +307,10 @@ public class IoTDBConfig {
*/
private long closeTsFileIntervalAfterFlushing = 60 * 60 * 1000L;
- /**
- * The interval to check whether tsfiles need closing. Unit: ms
- */
+ /** The interval to check whether tsfiles need closing. Unit: ms */
private long closeTsFileCheckInterval = 10 * 60 * 1000L;
- /**
- * When average series point number reaches this, flush the memtable to disk
- */
+ /** When average series point number reaches this, flush the memtable to disk */
private int avgSeriesPointNumberThreshold = 10000;
/**
@@ -433,9 +327,7 @@ public class IoTDBConfig {
*/
private int mergePagePointNumberThreshold = 100;
- /**
- * LEVEL_COMPACTION, NO_COMPACTION
- */
+ /** LEVEL_COMPACTION, NO_COMPACTION */
private CompactionStrategy compactionStrategy = CompactionStrategy.LEVEL_COMPACTION;
/**
@@ -457,9 +349,7 @@ public class IoTDBConfig {
*/
private int seqFileNumInEachLevel = 6;
- /**
- * Works when the compaction_strategy is LEVEL_COMPACTION. The max num of seq level.
- */
+ /** Works when the compaction_strategy is LEVEL_COMPACTION. The max num of seq level. */
private int seqLevelNum = 3;
/**
@@ -469,9 +359,7 @@ public class IoTDBConfig {
*/
private int unseqFileNumInEachLevel = 10;
- /**
- * Works when the compaction_strategy is LEVEL_COMPACTION. The max num of unseq level.
- */
+ /** Works when the compaction_strategy is LEVEL_COMPACTION. The max num of unseq level. */
private int unseqLevelNum = 1;
/**
@@ -482,49 +370,31 @@ public class IoTDBConfig {
*/
private int maxSelectUnseqFileNumInEachUnseqCompaction = 2000;
- /**
- * whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
- */
+ /** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
private boolean metaDataCacheEnable = true;
- /**
- * Memory allocated for timeSeriesMetaData cache in read process
- */
+ /** Memory allocated for timeSeriesMetaData cache in read process */
private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 5;
- /**
- * Memory allocated for chunk cache in read process
- */
+ /** Memory allocated for chunk cache in read process */
private long allocateMemoryForChunkCache = allocateMemoryForRead / 10;
- /**
- * Whether to enable Last cache
- */
+ /** Whether to enable Last cache */
private boolean lastCacheEnable = true;
- /**
- * Set true to enable statistics monitor service, false to disable statistics service.
- */
+ /** Set true to enable statistics monitor service, false to disable statistics service. */
private boolean enableStatMonitor = false;
- /**
- * Set true to enable writing monitor time series.
- */
+ /** Set true to enable writing monitor time series. */
private boolean enableMonitorSeriesWrite = false;
- /**
- * Cache size of {@code checkAndGetDataTypeCache} in {@link MManager}.
- */
+ /** Cache size of {@code checkAndGetDataTypeCache} in {@link MManager}. */
private int mManagerCacheSize = 300000;
- /**
- * Cache size of {@code checkAndGetDataTypeCache} in {@link MManager}.
- */
+ /** Cache size of {@code checkAndGetDataTypeCache} in {@link MManager}. */
private int mRemoteSchemaCacheSize = 100000;
- /**
- * Is external sort enable.
- */
+ /** Is external sort enable. */
private boolean enableExternalSort = true;
/**
@@ -533,13 +403,9 @@ public class IoTDBConfig {
*/
private int externalSortThreshold = 1000;
- /**
- * Is this IoTDB instance a receiver of sync or not.
- */
+ /** Is this IoTDB instance a receiver of sync or not. */
private boolean isSyncEnable = false;
- /**
- * If this IoTDB instance is a receiver of sync, set the server port.
- */
+ /** If this IoTDB instance is a receiver of sync, set the server port. */
private int syncServerPort = 5555;
/**
* Set the language version when loading file including error information, default value is "EN"
@@ -547,85 +413,53 @@ public class IoTDBConfig {
private String languageVersion = "EN";
private String ipWhiteList = "0.0.0.0/0";
- /**
- * Examining period of cache file reader : 100 seconds.
- */
+ /** Examining period of cache file reader : 100 seconds. */
private long cacheFileReaderClearPeriod = 100000;
- /**
- * the max executing time of query in ms.
- */
+ /** the max executing time of query in ms. */
private int queryTimeoutThreshold = 60000;
- /**
- * compaction interval in ms
- */
+ /** compaction interval in ms */
private long compactionInterval = 30000;
- /**
- * Replace implementation class of JDBC service
- */
+ /** Replace implementation class of JDBC service */
private String rpcImplClassName = TSServiceImpl.class.getName();
- /**
- * Is stat performance of sub-module enable.
- */
+ /** Is stat performance of sub-module enable. */
private boolean enablePerformanceStat = false;
- /**
- * Is performance tracing enable.
- */
+ /** Is performance tracing enable. */
private boolean enablePerformanceTracing = false;
private boolean enableCPV = false;
- /**
- * The display of stat performance interval in ms.
- */
+ /** The display of stat performance interval in ms. */
private long performanceStatDisplayInterval = 60000;
- /**
- * The memory used for stat performance.
- */
+ /** The memory used for stat performance. */
private int performanceStatMemoryInKB = 20;
- /**
- * whether use chunkBufferPool.
- */
+ /** whether use chunkBufferPool. */
private boolean chunkBufferPoolEnable = false;
- /**
- * Switch of watermark function
- */
+ /** Switch of watermark function */
private boolean enableWatermark = false;
- /**
- * Secret key for watermark
- */
+ /** Secret key for watermark */
private String watermarkSecretKey = "IoTDB*2019@Beijing";
- /**
- * Bit string of watermark
- */
+ /** Bit string of watermark */
private String watermarkBitString = "100101110100";
- /**
- * Watermark method and parameters
- */
+ /** Watermark method and parameters */
private String watermarkMethod = "GroupBasedLSBMethod(embed_row_cycle=2,embed_lsb_num=5)";
- /**
- * Switch of creating schema automatically
- */
+ /** Switch of creating schema automatically */
private boolean enableAutoCreateSchema = true;
- /**
- * register time series as which type when receiving boolean string "true" or "false"
- */
+ /** register time series as which type when receiving boolean string "true" or "false" */
private TSDataType booleanStringInferType = TSDataType.BOOLEAN;
- /**
- * register time series as which type when receiving an integer string "67"
- */
+ /** register time series as which type when receiving an integer string "67" */
private TSDataType integerStringInferType = TSDataType.FLOAT;
/**
@@ -634,9 +468,7 @@ public class IoTDBConfig {
*/
private TSDataType longStringInferType = TSDataType.DOUBLE;
- /**
- * register time series as which type when receiving a floating number string "6.7"
- */
+ /** register time series as which type when receiving a floating number string "6.7" */
private TSDataType floatingStringInferType = TSDataType.FLOAT;
/**
@@ -645,59 +477,37 @@ public class IoTDBConfig {
*/
private TSDataType nanStringInferType = TSDataType.DOUBLE;
- /**
- * Storage group level when creating schema automatically is enabled
- */
+ /** Storage group level when creating schema automatically is enabled */
private int defaultStorageGroupLevel = 1;
- /**
- * BOOLEAN encoding when creating schema automatically is enabled
- */
+ /** BOOLEAN encoding when creating schema automatically is enabled */
private TSEncoding defaultBooleanEncoding = TSEncoding.RLE;
- /**
- * INT32 encoding when creating schema automatically is enabled
- */
+ /** INT32 encoding when creating schema automatically is enabled */
private TSEncoding defaultInt32Encoding = TSEncoding.RLE;
- /**
- * INT64 encoding when creating schema automatically is enabled
- */
+ /** INT64 encoding when creating schema automatically is enabled */
private TSEncoding defaultInt64Encoding = TSEncoding.RLE;
- /**
- * FLOAT encoding when creating schema automatically is enabled
- */
+ /** FLOAT encoding when creating schema automatically is enabled */
private TSEncoding defaultFloatEncoding = TSEncoding.GORILLA;
- /**
- * DOUBLE encoding when creating schema automatically is enabled
- */
+ /** DOUBLE encoding when creating schema automatically is enabled */
private TSEncoding defaultDoubleEncoding = TSEncoding.GORILLA;
- /**
- * TEXT encoding when creating schema automatically is enabled
- */
+ /** TEXT encoding when creating schema automatically is enabled */
private TSEncoding defaultTextEncoding = TSEncoding.PLAIN;
- /**
- * How much memory (in byte) can be used by a single merge task.
- */
+ /** How much memory (in byte) can be used by a single merge task. */
private long mergeMemoryBudget = (long) (Runtime.getRuntime().maxMemory() * 0.1);
- /**
- * How many threads will be set up to perform upgrade tasks.
- */
+ /** How many threads will be set up to perform upgrade tasks. */
private int upgradeThreadNum = 1;
- /**
- * How many threads will be set up to perform main merge tasks.
- */
+ /** How many threads will be set up to perform main merge tasks. */
private int mergeThreadNum = 1;
- /**
- * How many threads will be set up to perform unseq merge chunk sub-tasks.
- */
+ /** How many threads will be set up to perform unseq merge chunk sub-tasks. */
private int mergeChunkSubThreadNum = 4;
/**
@@ -726,9 +536,7 @@ public class IoTDBConfig {
*/
private boolean forceFullMerge = true;
- /**
- * The limit of compaction merge can reach per second
- */
+ /** The limit of compaction merge can reach per second */
private int mergeWriteThroughputMbPerSec = 8;
/**
@@ -739,80 +547,53 @@ public class IoTDBConfig {
private MergeFileStrategy mergeFileStrategy = MergeFileStrategy.MAX_SERIES_NUM;
- /**
- * Default system file storage is in local file system (unsupported)
- */
+ /** Default system file storage is in local file system (unsupported) */
private FSType systemFileStorageFs = FSType.LOCAL;
- /**
- * Default TSfile storage is in local file system
- */
+ /** Default TSfile storage is in local file system */
private FSType tsFileStorageFs = FSType.LOCAL;
- /**
- * Default core-site.xml file path is /etc/hadoop/conf/core-site.xml
- */
+ /** Default core-site.xml file path is /etc/hadoop/conf/core-site.xml */
private String coreSitePath = "/etc/hadoop/conf/core-site.xml";
- /**
- * Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml
- */
+ /** Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml */
private String hdfsSitePath = "/etc/hadoop/conf/hdfs-site.xml";
- /**
- * Default HDFS ip is localhost
- */
+ /** Default HDFS ip is localhost */
private String hdfsIp = "localhost";
- /**
- * Default HDFS port is 9000
- */
+ /** Default HDFS port is 9000 */
private String hdfsPort = "9000";
- /**
- * Default DFS NameServices is hdfsnamespace
- */
+ /** Default DFS NameServices is hdfsnamespace */
private String dfsNameServices = "hdfsnamespace";
- /**
- * Default DFS HA name nodes are nn1 and nn2
- */
+ /** Default DFS HA name nodes are nn1 and nn2 */
private String dfsHaNamenodes = "nn1,nn2";
- /**
- * Default DFS HA automatic failover is enabled
- */
+ /** Default DFS HA automatic failover is enabled */
private boolean dfsHaAutomaticFailoverEnabled = true;
/**
- * Default DFS client failover proxy provider is "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ * Default DFS client failover proxy provider is
+ * "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
*/
private String dfsClientFailoverProxyProvider =
"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
- /**
- * whether use kerberos to authenticate hdfs
- */
+ /** whether use kerberos to authenticate hdfs */
private boolean useKerberos = false;
- /**
- * full path of kerberos keytab file
- */
+ /** full path of kerberos keytab file */
private String kerberosKeytabFilePath = "/path";
- /**
- * kerberos principal
- */
+ /** kerberos principal */
private String kerberosPrincipal = "your principal";
- /**
- * the num of memtable in each storage group
- */
+ /** the num of memtable in each storage group */
private int concurrentWritingTimePartition = 500;
- /**
- * the default fill interval in LinearFill and PreviousFill, -1 means infinite past time
- */
+ /** the default fill interval in LinearFill and PreviousFill, -1 means infinite past time */
private int defaultFillInterval = -1;
/**
@@ -823,24 +604,16 @@ public class IoTDBConfig {
*/
private long defaultTTL = Long.MAX_VALUE;
- /**
- * The default value of primitive array size in array pool
- */
+ /** The default value of primitive array size in array pool */
private int primitiveArraySize = 32;
- /**
- * whether enable data partition. If disabled, all data belongs to partition 0
- */
+ /** whether enable data partition. If disabled, all data belongs to partition 0 */
private boolean enablePartition = false;
- /**
- * whether enable MTree snapshot
- */
+ /** whether enable MTree snapshot */
private boolean enableMTreeSnapshot = false;
- /**
- * Interval line number of mlog.txt when creating a checkpoint and saving snapshot of mtree
- */
+ /** Interval line number of mlog.txt when creating a checkpoint and saving snapshot of mtree */
private int mtreeSnapshotInterval = 100000;
/**
@@ -849,9 +622,7 @@ public class IoTDBConfig {
*/
private int mtreeSnapshotThresholdTime = 3600;
- /**
- * Time range for partitioning data inside each storage group, the unit is second
- */
+ /** Time range for partitioning data inside each storage group, the unit is second */
private long partitionInterval = 604800;
/**
@@ -911,14 +682,10 @@ public class IoTDBConfig {
private int thriftDefaultBufferSize = RpcUtils.THRIFT_DEFAULT_BUF_CAPACITY;
- /**
- * time interval in minute for calculating query frequency
- */
+ /** time interval in minute for calculating query frequency */
private int frequencyIntervalInMinute = 1;
- /**
- * time cost(ms) threshold for slow query
- */
+ /** time cost(ms) threshold for slow query */
private long slowQueryThreshold = 5000;
/**
@@ -927,14 +694,10 @@ public class IoTDBConfig {
*/
private boolean enableRpcService = true;
- /**
- * the size of ioTaskQueue
- */
+ /** the size of ioTaskQueue */
private int ioTaskQueueSizeForFlushing = 10;
- /**
- * the number of virtual storage groups per user-defined storage group
- */
+ /** the number of virtual storage groups per user-defined storage group */
private int virtualStorageGroupNum = 1;
private String adminName = "root";
@@ -1055,9 +818,7 @@ public class IoTDBConfig {
confirmMultiDirStrategy();
}
- /**
- * if the folders are relative paths, add IOTDB_HOME as the path prefix
- */
+ /** if the folders are relative paths, add IOTDB_HOME as the path prefix */
private void formulateFolders() {
systemDir = addHomeDir(systemDir);
schemaDir = addHomeDir(schemaDir);
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 9f7bc94f94..2963fbaf77 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -18,16 +18,6 @@
*/
package org.apache.iotdb.db.conf;
-import com.google.common.net.InetAddresses;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.Properties;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -37,9 +27,21 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
+
+import com.google.common.net.InetAddresses;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.Properties;
+
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
@@ -105,9 +107,7 @@ public class IoTDBDescriptor {
}
}
- /**
- * load an property file and set TsfileDBConfig variables.
- */
+ /** load an property file and set TsfileDBConfig variables. */
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
private void loadProps() {
URL url = getPropsUrl();
@@ -1284,9 +1284,7 @@ public class IoTDBDescriptor {
}
}
- /**
- * Get default encode algorithm by data type
- */
+ /** Get default encode algorithm by data type */
public TSEncoding getDefaultEncodingByType(TSDataType dataType) {
switch (dataType) {
case BOOLEAN:
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 7090e47b99..f5057f9bf1 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -63,9 +63,7 @@ import java.util.concurrent.TimeUnit;
import static org.junit.Assert.fail;
-/**
- * This class is used for cleaning test environment in unit test and integration test
- */
+/** This class is used for cleaning test environment in unit test and integration test */
public class EnvironmentUtils {
private static final Logger logger = LoggerFactory.getLogger(EnvironmentUtils.class);
@@ -156,7 +154,7 @@ public class EnvironmentUtils {
MemTableManager.getInstance().close();
// delete all directory
- cleanAllDir();
+ // cleanAllDir();
config.setSeqTsFileSize(oldSeqTsFileSize);
config.setUnSeqTsFileSize(oldUnSeqTsFileSize);
config.setMemtableSizeThreshold(oldGroupSizeInByte);
@@ -235,22 +233,19 @@ public class EnvironmentUtils {
FileUtils.deleteDirectory(new File(dir));
}
- /**
- * disable the system monitor</br> this function should be called before all code in the setup
- */
+ /** disable the system monitor</br> this function should be called before all code in the setup */
public static void closeStatMonitor() {
config.setEnableStatMonitor(false);
}
- /**
- * disable memory control</br> this function should be called before all code in the setup
- */
+ /** disable memory control</br> this function should be called before all code in the setup */
public static void envSetUp() {
logger.warn("EnvironmentUtil setup...");
IoTDBDescriptor.getInstance().getConfig().setThriftServerAwaitTimeForStopService(60);
// we do not start 8181 port in test.
-// IoTDBDescriptor.getInstance().getConfig().setEnableMetricService(false);
-// IoTDBDescriptor.getInstance().getConfig().setAvgSeriesPointNumberThreshold(Integer.MAX_VALUE);
+ // IoTDBDescriptor.getInstance().getConfig().setEnableMetricService(false);
+ //
+ // IoTDBDescriptor.getInstance().getConfig().setAvgSeriesPointNumberThreshold(Integer.MAX_VALUE);
if (daemon == null) {
daemon = new IoTDB();
}
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
index 332b8c9b99..471417f958 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
@@ -1,15 +1,5 @@
package org.apache.iotdb.session;
-import static org.junit.Assert.assertEquals;
-
-import com.nimbusds.jose.util.ByteUtils;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -20,12 +10,20 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
public class MyRealDataWriteQueryTest {
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -43,32 +41,35 @@ public class MyRealDataWriteQueryTest {
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
-
private static String device = "root.game";
private static String measurement = "s6";
- private static TSDataType tsDataType = TSDataType.INT64;// TSDataType.DOUBLE;
+ private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
private static String timestamp_precision = "ns"; // ns, us, ms
// used to bound tqs random position
private static long dataMinTime = 0;
- private static long dataMaxTime = 511483966L;// 617426057626L;
+ private static long dataMaxTime = 617426057626L; // 511483966L;// 617426057626L;
private static long total_time_length =
dataMaxTime - dataMinTime; // in corresponding timestamp precision
- private static int total_point_number = 1000; // 1200000;
- private static int iotdb_chunk_point_size = 50;
+ private static int total_point_number = 1200000; // 1000; // 1200000;
+ private static int iotdb_chunk_point_size = 100;
private static long chunkAvgTimeLen =
- (long) Math
- .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
- private static String filePath = "D:\\github\\m4-lsm\\iotdb\\session\\src\\test\\java\\org\\apache\\iotdb\\session\\BallSpeedSmallData.csv";
- private static int deletePercentage = 2; // 0 means no deletes. 0-100
- private static int deleteLenPercentage = 50; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
- private static int timeIdx = 0; // 时间戳idx，从0开始
- private static int valueIdx = 1; // 值idx，从0开始
+ (long)
+ Math.ceil(
+ total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+ // private static String filePath =
+ // "D:\\github\\m4-lsm\\iotdb\\session\\src\\test\\java\\org\\apache\\iotdb\\session\\BallSpeedSmallData.csv";
+ private static String filePath =
+ "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+ private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ private static int timeIdx = 0; // 时间戳idx，从0开始
+ private static int valueIdx = 1; // 值idx，从0开始
@Before
public void setUp() throws Exception {
config.setEnableCPV(true);
config.setTimestampPrecision(timestamp_precision);
- config.setAvgSeriesPointNumberThreshold(100);
+ config.setAvgSeriesPointNumberThreshold(iotdb_chunk_point_size);
config.setUnSeqTsFileSize(1073741824);
config.setSeqTsFileSize(1073741824);
config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
@@ -76,7 +77,8 @@ public class MyRealDataWriteQueryTest {
config.setEnablePerformanceStat(false);
TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(1073741824);
- TSFileDescriptor.getInstance().getConfig().setEnableRegularityTimeDecode(true);
+
+ TSFileDescriptor.getInstance().getConfig().setEnableRegularityTimeDecode(false);
TSFileDescriptor.getInstance().getConfig().setRegularTimeInterval(511996);
EnvironmentUtils.envSetUp(); // start after configuration settings
@@ -97,17 +99,20 @@ public class MyRealDataWriteQueryTest {
System.out.println("[WriteData] deleteLenPercentage=" + deleteLenPercentage);
System.out.println("[WriteData] timeIdx=" + timeIdx);
System.out.println("[WriteData] valueIdx=" + valueIdx);
+ System.out.println(
+ "[WriteData] enableRegularityTimeDecode="
+ + TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode());
}
@After
public void tearDown() throws Exception {
- EnvironmentUtils.cleanEnv();
+ EnvironmentUtils.cleanEnv(); // comment out the cleanAllDir()
}
- /**
- * Before writing data, make sure check the server parameter configurations.
- */
- // Usage: java -jar WriteData-0.12.4.jar device measurement dataType timestamp_precision total_time_length total_point_number iotdb_chunk_point_size filePath deleteFreq deleteLen timeIdx valueIdx
+ /** Before writing data, make sure check the server parameter configurations. */
+ // Usage: java -jar WriteData-0.12.4.jar device measurement dataType timestamp_precision
+ // total_time_length total_point_number iotdb_chunk_point_size filePath deleteFreq deleteLen
+ // timeIdx valueIdx
@Test
public void test1() throws Exception {
System.out.println("writing data...");
@@ -118,7 +123,7 @@ public class MyRealDataWriteQueryTest {
// [tqs,tqe) range length, i.e., tqe-tqs
long range = total_time_length;
// w数量
- int w = 2;
+ int w = 1000;
System.out.println("[QueryData] query range=" + range);
System.out.println("[QueryData] w=" + w);
@@ -165,8 +170,7 @@ public class MyRealDataWriteQueryTest {
String sql;
if (approach.equals("mac")) {
// MAC UDF
- sql =
- String.format(queryFormat_UDF, measurement, device, minTime, maxTime, w); // MAC
+ sql = String.format(queryFormat_UDF, measurement, device, minTime, maxTime, w); // MAC
} else {
// MOC and CPV sql use the same sql queryFormat.
sql =
@@ -182,40 +186,40 @@ public class MyRealDataWriteQueryTest {
minTime,
maxTime,
interval,
- timestamp_precision); // note the time precision unit
+ timestamp_precision); // note the time precision unit
}
System.out.println("[QueryData] sql=" + sql);
long c = 0;
SessionDataSet dataSet = session.executeQueryStatement(sql);
DataIterator iterator = dataSet.iterator();
- System.out.println(dataSet.getColumnNames());
+ // System.out.println(dataSet.getColumnNames());
while (iterator.next()) { // this way avoid constructing rowRecord
c++;
- String ans;
- if (approach.equals("mac")) {
- ans =
- String.format(
- "%s,%s",
- iterator.getString(1), // time
- iterator.getString(2)); // M4
- } else {
- ans =
- String.format(
- "%s,%s,%s,%s,%s,%s,%s",
- iterator.getString(1), // time
- iterator.getString(2), // min_time
- iterator.getString(3), // max_time
- iterator.getString(4), // first_value
- iterator.getString(5), // last_value
- iterator.getString(6), // min_value & bottomTime
- iterator.getString(7)); // max_value & topTime
- }
- System.out.println(ans);
+ // String ans;
+ // if (approach.equals("mac")) {
+ // ans =
+ // String.format(
+ // "%s,%s",
+ // iterator.getString(1), // time
+ // iterator.getString(2)); // M4
+ // } else {
+ // ans =
+ // String.format(
+ // "%s,%s,%s,%s,%s,%s,%s",
+ // iterator.getString(1), // time
+ // iterator.getString(2), // min_time
+ // iterator.getString(3), // max_time
+ // iterator.getString(4), // first_value
+ // iterator.getString(5), // last_value
+ // iterator.getString(6), // min_value & bottomTime
+ // iterator.getString(7)); // max_value & topTime
+ // }
+ // System.out.println(ans);
}
Assert.assertEquals(w, c);
-// session.executeNonQueryStatement("clear cache");
+ // session.executeNonQueryStatement("clear cache");
dataSet = session.executeFinish();
String info = dataSet.getFinishResult();
System.out.println(
@@ -236,8 +240,12 @@ public class MyRealDataWriteQueryTest {
}
int deletePeriod =
- (int) Math.floor(100 * 1.0 / deletePercentage
- * iotdb_chunk_point_size); // use period to control percentage
+ (int)
+ Math.floor(
+ 100
+ * 1.0
+ / deletePercentage
+ * iotdb_chunk_point_size); // use period to control percentage
long deleteLen = (long) Math.floor(chunkAvgTimeLen * deleteLenPercentage * 1.0 / 100);
List<String> deletePaths = new ArrayList<>();
@@ -330,4 +338,3 @@ public class MyRealDataWriteQueryTest {
}
}
}
-
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index a435c7f4bb..7a6ab3e41a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -18,11 +18,12 @@
*/
package org.apache.iotdb.tsfile.common.conf;
-import java.io.Serializable;
-import java.nio.charset.Charset;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.fileSystem.FSType;
+import java.io.Serializable;
+import java.nio.charset.Charset;
+
/** TSFileConfig is a configure class. Every variables is public and has default value. */
public class TSFileConfig implements Serializable {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index b97bd9a9d7..0eb4fb7dac 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -19,6 +19,12 @@
package org.apache.iotdb.tsfile.common.conf;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.apache.iotdb.tsfile.utils.Loader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@ -27,10 +33,6 @@ import java.io.InputStream;
import java.net.URL;
import java.util.Properties;
import java.util.Set;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.utils.Loader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/** TSFileDescriptor is used to load TSFileConfig and provide configure information. */
public class TSFileDescriptor {
@@ -105,12 +107,18 @@ public class TSFileDescriptor {
properties.load(inputStream);
conf.setEnableRegularityTimeDecode(
Boolean.parseBoolean(
- properties.getProperty("enable_regularity_time_decode",
- Boolean.toString(conf.isEnableRegularityTimeDecode())).trim()));
-
- conf.setRegularTimeInterval(Long.parseLong(properties
- .getProperty("regular_time_interval", Long.toString(conf.getRegularTimeInterval()))
- .trim()));
+ properties
+ .getProperty(
+ "enable_regularity_time_decode",
+ Boolean.toString(conf.isEnableRegularityTimeDecode()))
+ .trim()));
+
+ conf.setRegularTimeInterval(
+ Long.parseLong(
+ properties
+ .getProperty(
+ "regular_time_interval", Long.toString(conf.getRegularTimeInterval()))
+ .trim()));
conf.setGroupSizeInByte(
Integer.parseInt(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index 20c8a6198d..2791149389 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -19,17 +19,19 @@
package org.apache.iotdb.tsfile.encoding.decoder;
-import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
/**
* This class is a decoder for decoding the byte array that encoded by {@code
- * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long values.<br> .
+ * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long values.<br>
+ * .
*
* @see DeltaBinaryEncoder
*/
@@ -38,24 +40,16 @@ public abstract class DeltaBinaryDecoder extends Decoder {
protected long count = 0;
protected byte[] deltaBuf;
- /**
- * the first value in one pack.
- */
+ /** the first value in one pack. */
protected int readIntTotalCount = 0;
protected int nextReadIndex = 0;
- /**
- * max bit length of all value in a pack.
- */
+ /** max bit length of all value in a pack. */
protected int packWidth;
- /**
- * data number in this pack.
- */
+ /** data number in this pack. */
protected int packNum;
- /**
- * how many bytes data takes after encoding.
- */
+ /** how many bytes data takes after encoding. */
protected int encodingLength;
public DeltaBinaryDecoder() {
@@ -88,9 +82,7 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private int firstValue;
private int[] data;
private int previous;
- /**
- * minimum value for all difference.
- */
+ /** minimum value for all difference. */
private int minDeltaBase;
public IntDeltaDecoder() {
@@ -174,21 +166,20 @@ public abstract class DeltaBinaryDecoder extends Decoder {
private long firstValue;
private long[] data;
private long previous;
- /**
- * minimum value for all difference.
- */
+ /** minimum value for all difference. */
private long minDeltaBase;
private boolean enableRegularityTimeDecode;
private long regularTimeInterval;
- private byte[] encodedRegularTimeInterval; // it depends on minDeltaBase and bitWidth of each pack
+ private byte[]
+ encodedRegularTimeInterval; // it depends on minDeltaBase and bitWidth of each pack
public LongDeltaDecoder() {
super();
- this.enableRegularityTimeDecode = TSFileDescriptor.getInstance().getConfig()
- .isEnableRegularityTimeDecode();
- this.regularTimeInterval = TSFileDescriptor.getInstance().getConfig()
- .getRegularTimeInterval();
+ this.enableRegularityTimeDecode =
+ TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode();
+ this.regularTimeInterval =
+ TSFileDescriptor.getInstance().getConfig().getRegularTimeInterval();
}
/**
@@ -228,7 +219,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
if (enableRegularityTimeDecode) {
// TODO encode regular time interval
long newDelta = regularTimeInterval - minDeltaBase;
- int bitWidthToByteNum = ceil(packWidth);
+ int bitWidthToByteNum =
+ ceil(packWidth); // packWidth has been made to be a multiple of 8 when encoding
encodedRegularTimeInterval = new byte[bitWidthToByteNum];
BytesUtils.longToBytes(newDelta, encodedRegularTimeInterval, 0, packWidth);
@@ -237,17 +229,22 @@ public abstract class DeltaBinaryDecoder extends Decoder {
// (2) compare bits with encodedRegularTimeInterval,
// (3) equal to reuse, else to convert
- long v = 0;
- int temp = 0;
- int pos = packWidth * i;
- for (int j = 0; j < packWidth; j++) {
- temp = (pos + packWidth - 1 - j) / 8;
- int bit = BytesUtils.getByteN(deltaBuf[temp], pos + packWidth - 1 - j);
- // 的确如果这样一位位地比较，就没有加速意义了。
- v = BytesUtils.setLongN(v, j, bit);
+ boolean equal = true;
+ int pos = i * bitWidthToByteNum;
+ for (int j = 0; j < bitWidthToByteNum; j++) { // compare encoded bytes
+ byte regular = encodedRegularTimeInterval[j];
+ byte data = deltaBuf[pos + j];
+ if (regular != data) {
+ equal = false;
+ break;
+ }
+ }
+ if (equal) {
+ data[i] = previous + regularTimeInterval;
+ } else {
+ long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
+ data[i] = previous + minDeltaBase + v;
}
-
- data[i] = previous + minDeltaBase + v;
previous = data[i];
}
} else { // without regularity-aware decoding
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index 672484bf6f..c7c87584d4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -19,14 +19,17 @@
package org.apache.iotdb.tsfile.encoding.encoder;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
/**
* DeltaBinaryEncoder is a encoder for compressing data in type of integer and long. We adapt a
* hypothesis that contiguous data points have similar values. Thus the difference value of two
@@ -35,8 +38,8 @@ import org.slf4j.LoggerFactory;
* Given an array which length is n, if all values in input data array are all positive and less
* than 2^m, we need actually m*n, but not 32*n bits to store the array.
*
- * <p>DeltaBinaryEncoder calculates difference between two adjacent points and record the minimum
- * of those difference values firstly. Then it saves two_diff value that difference minus minimum of
+ * <p>DeltaBinaryEncoder calculates difference between two adjacent points and record the minimum of
+ * those difference values firstly. Then it saves two_diff value that difference minus minimum of
* them, to make sure all two_diff values are positive. Then it statistics the longest bit length
* {@code m} it takes for each two_diff value, which means the bit length that maximum two_diff
* value takes. Only the low m bits are saved into result byte array for all two_diff values.
@@ -73,9 +76,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
protected abstract int calculateBitWidthsForDeltaBlockBuffer();
- /**
- * write all data into {@code encodingBlockBuffer}.
- */
+ /** write all data into {@code encodingBlockBuffer}. */
private void writeDataWithMinWidth() {
for (int i = 0; i < writeIndex; i++) {
writeValueToBytes(i);
@@ -101,6 +102,12 @@ public abstract class DeltaBinaryEncoder extends Encoder {
calcTwoDiff(i);
}
writeWidth = calculateBitWidthsForDeltaBlockBuffer();
+
+ // TODO ceil writeWidth to be a multiple of 8
+ if (TSFileDescriptor.getInstance().getConfig().isEnableRegularityTimeDecode()) {
+ writeWidth = 8 * (int) Math.ceil((double) (writeWidth) / 8.0);
+ }
+
writeHeaderToBytes();
writeDataWithMinWidth();
@@ -108,9 +115,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
writeIndex = -1;
}
- /**
- * calling this method to flush all values which haven't encoded to result byte array.
- */
+ /** calling this method to flush all values which haven't encoded to result byte array. */
@Override
public void flush(ByteArrayOutputStream out) {
try {
@@ -164,7 +169,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
* input a integer.
*
* @param value value to encode
- * @param out the ByteArrayOutputStream which data encode into
+ * @param out the ByteArrayOutputStream which data encode into
*/
public void encodeValue(int value, ByteArrayOutputStream out) {
if (writeIndex == -1) {
@@ -310,7 +315,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
* input a integer or long value.
*
* @param value value to encode
- * @param out - the ByteArrayOutputStream which data encode into
+ * @param out - the ByteArrayOutputStream which data encode into
*/
public void encodeValue(long value, ByteArrayOutputStream out) {
if (writeIndex == -1) {