You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2021/12/09 09:19:27 UTC
[iotdb] branch xkf_id_table updated: fix last query and add tests
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch xkf_id_table
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/xkf_id_table by this push:
new 7daa508 fix last query and add tests
7daa508 is described below
commit 7daa508c881c209bbce7cca5935683894f4f801d
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Thu Dec 9 16:20:39 2021 +0800
fix last query and add tests
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 22 +--
.../org/apache/iotdb/db/engine/StorageEngine.java | 50 +++---
.../iotdb/db/engine/flush/MemTableFlushTask.java | 12 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 33 ++--
.../apache/iotdb/db/engine/memtable/IMemTable.java | 11 +-
.../db/engine/memtable/PrimitiveMemTable.java | 3 +-
.../engine/storagegroup/StorageGroupProcessor.java | 68 +++----
.../db/engine/storagegroup/TsFileProcessor.java | 61 ++++---
.../org/apache/iotdb/db/metadata/MManager.java | 48 ++---
.../apache/iotdb/db/metadata/id_table/IDTable.java | 42 ++++-
.../metadata/id_table/entry/DeviceIDFactory.java | 3 +-
.../id_table/entry/InsertMeasurementMNode.java | 3 +-
.../apache/iotdb/db/metadata/path/AlignedPath.java | 18 +-
.../iotdb/db/metadata/path/MeasurementPath.java | 14 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 3 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 102 +++++------
.../iotdb/db/query/executor/LastQueryExecutor.java | 72 +++++++-
.../iotdb/db/writelog/recover/LogReplayer.java | 14 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 12 +-
.../db/engine/memtable/MemTableTestUtils.java | 7 +-
.../db/engine/memtable/MemtableBenchmark.java | 3 +-
.../db/engine/memtable/PrimitiveMemTableTest.java | 14 +-
.../id_table/IDTableResourceControlTest.java | 45 ++++-
.../iotdb/db/metadata/id_table/IDTableTest.java | 20 ++-
.../metadata/id_table/InsertWithIDTableTest.java | 94 ++++++----
.../db/metadata/id_table/QueryWithIDTableTest.java | 197 +++++++++++++++++++++
.../db/metadata/id_table/entry/DeviceIDTest.java | 7 +-
28 files changed, 696 insertions(+), 294 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 38030e5..f61eaf8 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -18,11 +18,6 @@
*/
package org.apache.iotdb.db.conf;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
-
-import java.io.File;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.compaction.CompactionPriority;
import org.apache.iotdb.db.engine.compaction.cross.CrossCompactionStrategy;
@@ -40,9 +35,16 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSType;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
public class IoTDBConfig {
/* Names of Watermark methods */
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index a831a7e..ed712eb 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -18,16 +18,6 @@
*/
package org.apache.iotdb.db.conf;
-import com.google.common.net.InetAddresses;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.Properties;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.compaction.CompactionPriority;
@@ -41,9 +31,21 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
+
+import com.google.common.net.InetAddresses;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.Properties;
+
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 96a3f4b..063ab10 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -18,30 +18,6 @@
*/
package org.apache.iotdb.db.engine;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.ConcurrentModificationException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -93,9 +69,35 @@ import org.apache.iotdb.service.rpc.thrift.TSStatus;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
import org.apache.iotdb.tsfile.utils.Pair;
+
+import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
public class StorageEngine implements IService {
private static final Logger logger = LoggerFactory.getLogger(StorageEngine.class);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index c2d1f48..0e6ba9b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -18,11 +18,6 @@
*/
package org.apache.iotdb.db.engine.flush;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager;
@@ -34,9 +29,16 @@ import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+
/**
* flush task to flush one memtable using a pipeline model to flush, which is sort memtable ->
* encoding -> write to disk (io task)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index eb3d1f3..af9d984 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -18,17 +18,10 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.WriteProcessException;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.id_table.entry.DeviceIDFactory;
import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
@@ -39,9 +32,17 @@ import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
public abstract class AbstractMemTable implements IMemTable {
private final Map<IDeviceID, IWritableMemChunkGroup> memTableMap;
@@ -300,8 +301,8 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public boolean checkIfChunkDoesNotExist(String deviceId, String measurement) {
- IWritableMemChunkGroup memChunkGroup = memTableMap.get(getDeviceID(deviceId));
+ public boolean checkIfChunkDoesNotExist(IDeviceID deviceId, String measurement) {
+ IWritableMemChunkGroup memChunkGroup = memTableMap.get(deviceId);
if (null == memChunkGroup) {
return true;
}
@@ -309,8 +310,8 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public long getCurrentChunkPointNum(String deviceId, String measurement) {
- IWritableMemChunkGroup memChunkGroup = memTableMap.get(getDeviceID(deviceId));
+ public long getCurrentChunkPointNum(IDeviceID deviceId, String measurement) {
+ IWritableMemChunkGroup memChunkGroup = memTableMap.get(deviceId);
return memChunkGroup.getCurrentChunkPointNum(measurement);
}
@@ -442,16 +443,6 @@ public abstract class AbstractMemTable implements IMemTable {
return createdTime;
}
- private IDeviceID getDeviceID(String deviceId) {
- try {
- return DeviceIDFactory.getInstance().getDeviceID(new PartialPath(deviceId));
- } catch (IllegalPathException e) {
- logger.error("device id is illegal");
- }
-
- return null;
- }
-
private IDeviceID getDeviceID(PartialPath deviceId) {
return DeviceIDFactory.getInstance().getDeviceID(deviceId);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
index 703a044..6c7633a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
@@ -18,9 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.WriteProcessException;
@@ -33,6 +30,10 @@ import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
/**
* IMemTable is designed to store data points which are not flushed into TsFile yet. An instance of
* IMemTable maintains all series belonging to one StorageGroup, corresponding to one
@@ -151,10 +152,10 @@ public interface IMemTable {
void release();
/** must guarantee the device exists in the work memtable only used when mem control enabled */
- boolean checkIfChunkDoesNotExist(String deviceId, String measurement);
+ boolean checkIfChunkDoesNotExist(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
- long getCurrentChunkPointNum(String deviceId, String measurement);
+ long getCurrentChunkPointNum(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
void addTextDataSize(long textDataIncrement);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
index 367d5c6..b4d3b37 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
@@ -19,9 +19,10 @@
package org.apache.iotdb.db.engine.memtable;
+import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
+
import java.util.HashMap;
import java.util.Map;
-import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
public class PrimitiveMemTable extends AbstractMemTable {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 767d9c1..a23f1c1 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -18,39 +18,6 @@
*/
package org.apache.iotdb.db.engine.storagegroup;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
-import static org.apache.iotdb.db.engine.compaction.cross.inplace.task.CrossSpaceMergeTask.MERGE_SUFFIX;
-import static org.apache.iotdb.db.engine.compaction.inner.utils.SizeTieredCompactionLogger.COMPACTION_LOG_NAME;
-import static org.apache.iotdb.db.engine.storagegroup.TsFileResource.TEMP_SUFFIX;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Pattern;
-import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -117,9 +84,44 @@ import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
+import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Pattern;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.db.engine.compaction.cross.inplace.task.CrossSpaceMergeTask.MERGE_SUFFIX;
+import static org.apache.iotdb.db.engine.compaction.inner.utils.SizeTieredCompactionLogger.COMPACTION_LOG_NAME;
+import static org.apache.iotdb.db.engine.storagegroup.TsFileResource.TEMP_SUFFIX;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
/**
* For sequence data, a StorageGroupProcessor has some TsFileProcessors, in which there is only one
* TsFileProcessor in the working status. <br>
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 8e58983..ecf532e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -18,16 +18,6 @@
*/
package org.apache.iotdb.db.engine.storagegroup;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentLinkedDeque;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.CompressionRatio;
@@ -48,8 +38,11 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.UpdateEndTi
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.id_table.entry.DeviceIDFactory;
+import org.apache.iotdb.db.metadata.id_table.entry.IDeviceID;
import org.apache.iotdb.db.metadata.path.AlignedPath;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
@@ -74,9 +67,21 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
@SuppressWarnings("java:S1135") // ignore todos
public class TsFileProcessor {
@@ -340,13 +345,14 @@ public class TsFileProcessor {
long memTableIncrement = 0L;
long textDataIncrement = 0L;
long chunkMetadataIncrement = 0L;
- String deviceId = insertRowPlan.getDeviceId().getFullPath();
+ IDeviceID deviceID = getDeviceID(insertRowPlan.getDeviceId().getFullPath());
+
for (int i = 0; i < insertRowPlan.getDataTypes().length; i++) {
// skip failed Measurements
if (insertRowPlan.getDataTypes()[i] == null || insertRowPlan.getMeasurements()[i] == null) {
continue;
}
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, insertRowPlan.getMeasurements()[i])) {
+ if (workMemTable.checkIfChunkDoesNotExist(deviceID, insertRowPlan.getMeasurements()[i])) {
// ChunkMetadataIncrement
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(
@@ -355,7 +361,7 @@ public class TsFileProcessor {
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
- workMemTable.getCurrentChunkPointNum(deviceId, insertRowPlan.getMeasurements()[i]);
+ workMemTable.getCurrentChunkPointNum(deviceID, insertRowPlan.getMeasurements()[i]);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i])
@@ -378,8 +384,9 @@ public class TsFileProcessor {
long textDataIncrement = 0L;
long chunkMetadataIncrement = 0L;
AlignedWritableMemChunk alignedMemChunk = null;
- String deviceId = insertRowPlan.getDeviceId().getFullPath();
- if (workMemTable.checkIfChunkDoesNotExist(deviceId, AlignedPath.VECTOR_PLACEHOLDER)) {
+ IDeviceID deviceID = getDeviceID(insertRowPlan.getDeviceId().getFullPath());
+
+ if (workMemTable.checkIfChunkDoesNotExist(deviceID, AlignedPath.VECTOR_PLACEHOLDER)) {
// ChunkMetadataIncrement
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
@@ -388,13 +395,13 @@ public class TsFileProcessor {
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
- workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
+ workMemTable.getCurrentChunkPointNum(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes())
: 0;
alignedMemChunk =
- ((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId))
+ ((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceID))
.getAlignedMemChunk();
}
for (int i = 0; i < insertRowPlan.getDataTypes().length; i++) {
@@ -425,7 +432,7 @@ public class TsFileProcessor {
}
long[] memIncrements = new long[3]; // memTable, text, chunk metadata
- String deviceId = insertTabletPlan.getDeviceId().getFullPath();
+ IDeviceID deviceID = getDeviceID(insertTabletPlan.getDeviceId().getFullPath());
for (int i = 0; i < insertTabletPlan.getDataTypes().length; i++) {
// skip failed Measurements
@@ -435,7 +442,7 @@ public class TsFileProcessor {
if (dataType == null || column == null || measurement == null) {
continue;
}
- updateMemCost(dataType, measurement, deviceId, start, end, memIncrements, column);
+ updateMemCost(dataType, measurement, deviceID, start, end, memIncrements, column);
}
long memTableIncrement = memIncrements[0];
long textDataIncrement = memIncrements[1];
@@ -451,7 +458,7 @@ public class TsFileProcessor {
}
long[] memIncrements = new long[3]; // memTable, text, chunk metadata
- String deviceId = insertTabletPlan.getDeviceId().getFullPath();
+ IDeviceID deviceId = getDeviceID(insertTabletPlan.getDeviceId().getFullPath());
updateAlignedMemCost(
insertTabletPlan.getDataTypes(),
@@ -471,7 +478,7 @@ public class TsFileProcessor {
private void updateMemCost(
TSDataType dataType,
String measurement,
- String deviceId,
+ IDeviceID deviceId,
int start,
int end,
long[] memIncrements,
@@ -507,7 +514,7 @@ public class TsFileProcessor {
private void updateAlignedMemCost(
TSDataType[] dataTypes,
- String deviceId,
+ IDeviceID deviceId,
String[] measurementIds,
int start,
int end,
@@ -1360,6 +1367,16 @@ public class TsFileProcessor {
return shouldClose;
}
+ private IDeviceID getDeviceID(String deviceId) {
+ try {
+ return DeviceIDFactory.getInstance().getDeviceID(new PartialPath(deviceId));
+ } catch (IllegalPathException e) {
+ logger.error("device id is illegal");
+ }
+
+ return null;
+ }
+
@TestOnly
public IMemTable getWorkMemTable() {
return workMemTable;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8fd6237..eb63295 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -18,28 +18,6 @@
*/
package org.apache.iotdb.db.metadata;
-import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
-
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -116,11 +94,35 @@ import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.TimeseriesSchema;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
+import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
/**
* This class takes the responsibility of serialization of all the metadata info and persistent it
* into files. This class contains all the interfaces to modify the metadata for delta system. All
@@ -875,7 +877,7 @@ public class MManager {
}
}
- public IMNode getDeviceNodeWithAutoCreate(PartialPath path)
+ protected IMNode getDeviceNodeWithAutoCreate(PartialPath path)
throws MetadataException, IOException {
return getDeviceNodeWithAutoCreate(
path, config.isAutoCreateSchemaEnabled(), true, config.getDefaultStorageGroupLevel());
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java
index e2f9274..9c517d7 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/IDTable.java
@@ -19,9 +19,6 @@
package org.apache.iotdb.db.metadata.id_table;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
@@ -40,10 +37,16 @@ import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
/** id table belongs to a storage group and mapping timeseries path to it's schema */
public class IDTable {
@@ -217,6 +220,35 @@ public class IDTable {
}
/**
+ * get last cache of the timeseies
+ *
+ * @param timeseriesID timeseries ID of the timeseries
+ * @throws MetadataException if the timeseries is not exits
+ */
+ public synchronized TimeValuePair getLastCache(TimeseriesID timeseriesID)
+ throws MetadataException {
+ return getSchemaEntry(timeseriesID).getCachedLast();
+ }
+
+ /**
+ * update last cache of the timeseies
+ *
+ * @param timeseriesID timeseries ID of the timeseries
+ * @param pair last time value pair
+ * @param highPriorityUpdate is high priority update
+ * @param latestFlushedTime last flushed time
+ * @throws MetadataException if the timeseries is not exits
+ */
+ public synchronized void updateLastCache(
+ TimeseriesID timeseriesID,
+ TimeValuePair pair,
+ boolean highPriorityUpdate,
+ Long latestFlushedTime)
+ throws MetadataException {
+ getSchemaEntry(timeseriesID).updateCachedLast(pair, highPriorityUpdate, latestFlushedTime);
+ }
+
+ /**
* check whether a time series is exist if exist, check the type consistency if not exist, call
* MManager to create it
*
@@ -328,13 +360,13 @@ public class IDTable {
DeviceEntry deviceEntry = idTables[slot].get(deviceID);
if (deviceEntry == null) {
throw new MetadataException(
- "update non exist timeseries's latest flushed time, timeseries id is: " + timeseriesID);
+ "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID);
}
SchemaEntry schemaEntry = deviceEntry.getSchemaEntry(timeseriesID.getMeasurement());
if (schemaEntry == null) {
throw new MetadataException(
- "update non exist timeseries's latest flushed time, timeseries id is: " + timeseriesID);
+ "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID);
}
return schemaEntry;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java
index b7ccfe0..1fd2752 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDFactory.java
@@ -19,10 +19,11 @@
package org.apache.iotdb.db.metadata.id_table.entry;
-import java.util.function.Function;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.metadata.path.PartialPath;
+import java.util.function.Function;
+
/** factory to build device id according to configured algorithm */
public class DeviceIDFactory {
Function<PartialPath, IDeviceID> getDeviceIDFunction;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/InsertMeasurementMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/InsertMeasurementMNode.java
index 87a53c6..1bc38f9 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/InsertMeasurementMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/id_table/entry/InsertMeasurementMNode.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.db.metadata.id_table.entry;
-import java.util.Map;
import org.apache.iotdb.db.engine.trigger.executor.TriggerExecutor;
import org.apache.iotdb.db.metadata.lastCache.container.ILastCacheContainer;
import org.apache.iotdb.db.metadata.logfile.MLogWriter;
@@ -34,6 +33,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+import java.util.Map;
+
public class InsertMeasurementMNode implements IMeasurementMNode {
SchemaEntry schemaEntry;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index 7de5b20..f9e9184 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@ -19,14 +19,6 @@
package org.apache.iotdb.db.metadata.path;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunkGroup;
import org.apache.iotdb.db.engine.memtable.IMemTable;
@@ -63,9 +55,19 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
/**
* VectorPartialPath represents many fullPaths of aligned timeseries. In the AlignedPath, the nodes
* in PartialPath is deviceId e.g. VectorPartialPath nodes=root.sg1.alignedD1 measurementList=[s1,
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
index 56551d9..702513d 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
@@ -18,12 +18,6 @@
*/
package org.apache.iotdb.db.metadata.path;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
@@ -55,9 +49,17 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
public class MeasurementPath extends PartialPath {
private static final Logger logger = LoggerFactory.getLogger(MeasurementPath.class);
diff --git a/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index efc778c..4cb8b8c 100644
--- a/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -181,7 +181,8 @@ public class StatMonitor implements StatMonitorMBean, IService {
Collections.singletonList(TSDataType.INT64),
new QueryContext(queryId),
null,
- Collections.singletonMap(monitorSeries.getDevice(), measurementSet))
+ Collections.singletonMap(monitorSeries.getDevice(), measurementSet),
+ config.isEnableIDTable())
.get(0)
.right;
if (timeValuePair.getValue() != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 6a352fa..b726262 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -18,56 +18,6 @@
*/
package org.apache.iotdb.db.qp.executor;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CANCELLED;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CHILD_NODES;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CHILD_PATHS;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_COLUMN;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_EVERY_INTERVAL;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_FOR_INTERVAL;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_NAME;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_QUERY_SQL;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_TARGET_PATH;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_COUNT;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CREATED_TIME;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_DEVICES;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_DONE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_CLASS;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_NAME;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_TYPE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ITEM;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_LOCK_INFO;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PRIVILEGE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PROGRESS;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ROLE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_STORAGE_GROUP;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TASK_NAME;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TTL;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_USER;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_VALUE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDAF;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDTF;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDAF;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDTF;
-import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_NATIVE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.db.conf.IoTDBConstant.QUERY_ID;
-import static org.apache.iotdb.db.conf.IoTDBConstant.STATEMENT;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer;
@@ -209,9 +159,61 @@ import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CANCELLED;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CHILD_NODES;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CHILD_PATHS;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_COLUMN;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_EVERY_INTERVAL;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_FOR_INTERVAL;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_NAME;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_QUERY_SQL;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CONTINUOUS_QUERY_TARGET_PATH;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_COUNT;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CREATED_TIME;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_DEVICES;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_DONE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_CLASS;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_NAME;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_TYPE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ITEM;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_LOCK_INFO;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PRIVILEGE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PROGRESS;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ROLE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_STORAGE_GROUP;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TASK_NAME;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TTL;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_USER;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_VALUE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDAF;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDTF;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDAF;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDTF;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_NATIVE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
+import static org.apache.iotdb.db.conf.IoTDBConstant.QUERY_ID;
+import static org.apache.iotdb.db.conf.IoTDBConstant.STATEMENT;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
public class PlanExecutor implements IPlanExecutor {
private static final Logger logger = LoggerFactory.getLogger(PlanExecutor.class);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
index 7508a60..f0c90a2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
@@ -26,6 +26,8 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.id_table.IDTable;
+import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.metadata.path.PartialPath;
@@ -70,6 +72,11 @@ public class LastQueryExecutor {
private static final boolean CACHE_ENABLED =
IoTDBDescriptor.getInstance().getConfig().isLastCacheEnabled();
private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
+ // for test to reload this parameter after restart, it can't be static
+ private final boolean isIdTableEnabled =
+ IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+
+ private static final Logger logger = LoggerFactory.getLogger(LastQueryExecutor.class);
public LastQueryExecutor(LastQueryPlan lastQueryPlan) {
this.selectedSeries = lastQueryPlan.getDeduplicatedPaths();
@@ -138,7 +145,12 @@ public class LastQueryExecutor {
RawDataQueryPlan lastQueryPlan)
throws QueryProcessException, StorageEngineException, IOException {
return calculateLastPairForSeriesLocally(
- seriesPaths, dataTypes, context, expression, lastQueryPlan.getDeviceToMeasurements());
+ seriesPaths,
+ dataTypes,
+ context,
+ expression,
+ lastQueryPlan.getDeviceToMeasurements(),
+ isIdTableEnabled);
}
public static List<Pair<Boolean, TimeValuePair>> calculateLastPairForSeriesLocally(
@@ -146,7 +158,8 @@ public class LastQueryExecutor {
List<TSDataType> dataTypes,
QueryContext context,
IExpression expression,
- Map<String, Set<String>> deviceMeasurementsMap)
+ Map<String, Set<String>> deviceMeasurementsMap,
+ boolean isIdTableEnabled)
throws QueryProcessException, StorageEngineException, IOException {
List<LastCacheAccessor> cacheAccessors = new ArrayList<>();
Filter filter = (expression == null) ? null : ((GlobalTimeExpression) expression).getFilter();
@@ -161,7 +174,8 @@ public class LastQueryExecutor {
cacheAccessors,
nonCachedPaths,
nonCachedDataTypes,
- context.isDebug());
+ context.isDebug(),
+ isIdTableEnabled);
if (nonCachedPaths.isEmpty()) {
return resultContainer;
}
@@ -220,11 +234,16 @@ public class LastQueryExecutor {
List<LastCacheAccessor> cacheAccessors,
List<PartialPath> restPaths,
List<TSDataType> restDataType,
- boolean debugOn) {
+ boolean debugOn,
+ boolean isIdTableEnabled) {
List<Pair<Boolean, TimeValuePair>> resultContainer = new ArrayList<>();
if (CACHE_ENABLED) {
for (PartialPath path : seriesPaths) {
- cacheAccessors.add(new LastCacheAccessor(path));
+ if (isIdTableEnabled) {
+ cacheAccessors.add(new IDTableLastCacheAccessor(path));
+ } else {
+ cacheAccessors.add(new MManagerLastCacheAccessor(path));
+ }
}
} else {
for (int i = 0; i < seriesPaths.size(); i++) {
@@ -262,12 +281,18 @@ public class LastQueryExecutor {
return resultContainer;
}
- private static class LastCacheAccessor {
+ private interface LastCacheAccessor {
+ public TimeValuePair read();
+
+ public void write(TimeValuePair pair);
+ }
+
+ private static class MManagerLastCacheAccessor implements LastCacheAccessor {
private final MeasurementPath path;
private IMeasurementMNode node;
- LastCacheAccessor(PartialPath seriesPath) {
+ MManagerLastCacheAccessor(PartialPath seriesPath) {
this.path = (MeasurementPath) seriesPath;
}
@@ -299,6 +324,39 @@ public class LastQueryExecutor {
}
}
+ private static class IDTableLastCacheAccessor implements LastCacheAccessor {
+
+ private PartialPath fullPath;
+
+ IDTableLastCacheAccessor(PartialPath seriesPath) {
+ fullPath = seriesPath;
+ }
+
+ @Override
+ public TimeValuePair read() {
+ try {
+ IDTable table =
+ StorageEngine.getInstance().getProcessor(fullPath.getDevicePath()).getIdTable();
+ return table.getLastCache(new TimeseriesID(fullPath));
+ } catch (StorageEngineException | MetadataException e) {
+ logger.error("last query can't find storage group: path is: " + fullPath);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void write(TimeValuePair pair) {
+ try {
+ IDTable table =
+ StorageEngine.getInstance().getProcessor(fullPath.getDevicePath()).getIdTable();
+ table.updateLastCache(new TimeseriesID(fullPath), pair, false, Long.MIN_VALUE);
+ } catch (StorageEngineException | MetadataException e) {
+ logger.error("last query can't find storage group: path is: " + fullPath);
+ }
+ }
+ }
+
private static boolean satisfyFilter(Filter filter, TimeValuePair tvPair) {
return filter == null || filter.satisfy(tvPair.getTimestamp(), tvPair.getValue().getValue());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
index c5af46c..c45bcbe 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
@@ -19,12 +19,6 @@
package org.apache.iotdb.db.writelog.recover;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Supplier;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
@@ -50,9 +44,17 @@ import org.apache.iotdb.db.writelog.io.ILogReader;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
/**
* LogReplayer finds the logNode of the TsFile given by insertFilePath and logNodePrefix, reads the
* WALs from the logNode and redoes them into a given MemTable and ModificationFile.
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
index 373d247..acd5757 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
@@ -18,11 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
@@ -32,10 +27,17 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
public class MemTableFlushTaskTest {
private RestorableTsFileIOWriter writer;
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java
index 06adfbe..f138f42 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java
@@ -18,9 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.id_table.entry.DeviceIDFactory;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
@@ -35,6 +32,10 @@ import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
public class MemTableTestUtils {
public static String deviceId0 = "d0";
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java
index 81ba74b..ab0d4e2 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import java.util.Collections;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.id_table.entry.DeviceIDFactory;
import org.apache.iotdb.db.metadata.path.PartialPath;
@@ -26,6 +25,8 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+import java.util.Collections;
+
/** Memtable insert benchmark. Bench the Memtable and get its performance. */
public class MemtableBenchmark {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
index eaad206..ee28451 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
@@ -18,12 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@ -50,10 +44,18 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
public class PrimitiveMemTableTest {
double delta;
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java
index bae4e76..e24d463 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableResourceControlTest.java
@@ -1,11 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.iotdb.db.metadata.id_table;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup;
@@ -26,26 +38,43 @@ import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
public class IDTableResourceControlTest {
private final Planner processor = new Planner();
private boolean isEnableIDTable = false;
+ private String originalDeviceIDTransformationMethod = null;
+
@Before
- public void setUp() {
+ public void before() {
isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+ originalDeviceIDTransformationMethod =
+ IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+
IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+ IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
EnvironmentUtils.envSetUp();
}
@After
- public void tearDown() throws Exception {
+ public void clean() throws IOException, StorageEngineException {
IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
EnvironmentUtils.cleanEnv();
}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java
index 440fa14..bf05155 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/IDTableTest.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -60,14 +61,29 @@ public class IDTableTest {
private CompressionType compressionType;
+ private boolean isEnableIDTable = false;
+
+ private String originalDeviceIDTransformationMethod = null;
+
@Before
- public void setUp() {
+ public void before() {
compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor();
+ IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true);
+ isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+ originalDeviceIDTransformationMethod =
+ IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+ IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
EnvironmentUtils.envSetUp();
}
@After
- public void tearDown() throws Exception {
+ public void clean() throws IOException, StorageEngineException {
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
EnvironmentUtils.cleanEnv();
}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java
index 459fa2c..5d6cc6b 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/InsertWithIDTableTest.java
@@ -1,5 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.iotdb.db.metadata.id_table;
+import static org.junit.Assert.assertEquals;
+
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -42,17 +63,26 @@ public class InsertWithIDTableTest {
private boolean isEnableIDTable = false;
+ private String originalDeviceIDTransformationMethod = null;
+
@Before
public void before() {
IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true);
isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+ originalDeviceIDTransformationMethod =
+ IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+
IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+ IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
EnvironmentUtils.envSetUp();
}
@After
public void clean() throws IOException, StorageEngineException {
IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
EnvironmentUtils.cleanEnv();
}
@@ -67,10 +97,10 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(6, dataSet.getPaths().size());
+ assertEquals(6, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
}
}
@@ -83,15 +113,15 @@ public class InsertWithIDTableTest {
PlanExecutor executor = new PlanExecutor();
executor.insert(vectorRowPlan);
- Assert.assertEquals("[s1, s2, s3]", Arrays.toString(vectorRowPlan.getMeasurementMNodes()));
+ assertEquals("[s1, s2, s3]", Arrays.toString(vectorRowPlan.getMeasurementMNodes()));
QueryPlan queryPlan =
(QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1.GPS");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(1, dataSet.getPaths().size());
+ assertEquals(1, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(3, record.getFields().size());
+ assertEquals(3, record.getFields().size());
}
}
@@ -148,10 +178,10 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(6, dataSet.getPaths().size());
+ assertEquals(6, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
}
}
@@ -166,13 +196,13 @@ public class InsertWithIDTableTest {
plan1.serialize(byteBuffer);
byteBuffer.flip();
- Assert.assertEquals(PhysicalPlanType.INSERT.ordinal(), byteBuffer.get());
+ assertEquals(PhysicalPlanType.INSERT.ordinal(), byteBuffer.get());
InsertRowPlan plan2 = new InsertRowPlan();
plan2.deserialize(byteBuffer);
executor.insert(plan2);
- Assert.assertEquals(plan1, plan2);
+ assertEquals(plan1, plan2);
}
@Test
@@ -241,14 +271,14 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan =
(QueryPlan) processor.parseSQLToPhysicalPlan("select s1 from root.isp.d1.GPS");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(1, dataSet.getPaths().size());
+ assertEquals(1, dataSet.getPaths().size());
int count = 0;
while (dataSet.hasNext()) {
count++;
RowRecord record = dataSet.next();
- Assert.assertEquals(1, record.getFields().size());
+ assertEquals(1, record.getFields().size());
}
- Assert.assertEquals(1, count);
+ assertEquals(1, count);
}
@Test
@@ -295,10 +325,10 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(6, dataSet.getPaths().size());
+ assertEquals(6, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
}
}
@@ -355,11 +385,11 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(6, dataSet.getPaths().size());
+ assertEquals(6, dataSet.getPaths().size());
int rowNum = 0;
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
List<Field> fields = record.getFields();
for (int i = 0; i < 6; ++i) {
if (i % times.length == rowNum) {
@@ -388,15 +418,15 @@ public class InsertWithIDTableTest {
PlanExecutor executor = new PlanExecutor();
executor.insertTablet(tabletPlan);
- Assert.assertEquals("[s1, s2, s3]", Arrays.toString(tabletPlan.getMeasurementMNodes()));
+ assertEquals("[s1, s2, s3]", Arrays.toString(tabletPlan.getMeasurementMNodes()));
QueryPlan queryPlan =
(QueryPlan) processor.parseSQLToPhysicalPlan("select ** from root.isp.d1");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(1, dataSet.getPaths().size());
+ assertEquals(1, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(3, record.getFields().size());
+ assertEquals(3, record.getFields().size());
}
}
@@ -411,13 +441,13 @@ public class InsertWithIDTableTest {
plan1.serialize(byteBuffer);
byteBuffer.flip();
- Assert.assertEquals(PhysicalPlanType.BATCHINSERT.ordinal(), byteBuffer.get());
+ assertEquals(PhysicalPlanType.BATCHINSERT.ordinal(), byteBuffer.get());
InsertTabletPlan plan2 = new InsertTabletPlan();
plan2.deserialize(byteBuffer);
executor.insertTablet(plan2);
- Assert.assertEquals(plan1, plan2);
+ assertEquals(plan1, plan2);
}
@Test
@@ -439,13 +469,13 @@ public class InsertWithIDTableTest {
plan1.serialize(byteBuffer);
byteBuffer.flip();
- Assert.assertEquals(PhysicalPlanType.BATCHINSERT.ordinal(), byteBuffer.get());
+ assertEquals(PhysicalPlanType.BATCHINSERT.ordinal(), byteBuffer.get());
InsertTabletPlan plan2 = new InsertTabletPlan();
plan2.deserialize(byteBuffer);
executor.insertTablet(plan2);
- Assert.assertEquals(plan1, plan2);
+ assertEquals(plan1, plan2);
}
@Test
@@ -464,10 +494,10 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan =
(QueryPlan) processor.parseSQLToPhysicalPlan("select ** from root.isp.d1");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(3, dataSet.getPaths().size());
+ assertEquals(3, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
}
// test recover
@@ -484,10 +514,10 @@ public class InsertWithIDTableTest {
queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select ** from root.isp.d1");
dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(3, dataSet.getPaths().size());
+ assertEquals(3, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
}
}
@@ -507,16 +537,16 @@ public class InsertWithIDTableTest {
// nothing can be found when we not insert data
QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select ** from root.isp");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(0, dataSet.getPaths().size());
+ assertEquals(0, dataSet.getPaths().size());
executor.insertTablet(tabletPlan);
queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select ** from root.isp");
dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(3, dataSet.getPaths().size());
+ assertEquals(3, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(6, record.getFields().size());
+ assertEquals(6, record.getFields().size());
}
}
@@ -704,10 +734,10 @@ public class InsertWithIDTableTest {
QueryPlan queryPlan =
(QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.multi.**");
QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
- Assert.assertEquals(60, dataSet.getPaths().size());
+ assertEquals(60, dataSet.getPaths().size());
while (dataSet.hasNext()) {
RowRecord record = dataSet.next();
- Assert.assertEquals(60, record.getFields().size());
+ assertEquals(60, record.getFields().size());
}
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java
new file mode 100644
index 0000000..243be7c
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/QueryWithIDTableTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.id_table;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.id_table.entry.TimeseriesID;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.db.qp.Planner;
+import org.apache.iotdb.db.qp.executor.PlanExecutor;
+import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
+import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsBinary;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsBoolean;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsDouble;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsFloat;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsInt;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsLong;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class QueryWithIDTableTest {
+ private final Planner processor = new Planner();
+
+ private boolean isEnableIDTable = false;
+
+ private String originalDeviceIDTransformationMethod = null;
+
+ Set<String> retSet =
+ new HashSet<>(
+ Arrays.asList(
+ "113\troot.isp.d1.s3\t10003\tINT64",
+ "113\troot.isp.d1.s4\t103\tINT32",
+ "113\troot.isp.d1.s5\tfalse\tBOOLEAN",
+ "113\troot.isp.d1.s6\thh3\tTEXT",
+ "113\troot.isp.d1.s1\t4.0\tDOUBLE",
+ "113\troot.isp.d1.s2\t5.0\tFLOAT"));
+
+ @Before
+ public void before() {
+ IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true);
+ isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+ originalDeviceIDTransformationMethod =
+ IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+ IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
+ EnvironmentUtils.envSetUp();
+ }
+
+ @After
+ public void clean() throws IOException, StorageEngineException {
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void testLastCacheQuery()
+ throws QueryProcessException, MetadataException, InterruptedException,
+ QueryFilterOptimizationException, StorageEngineException, IOException {
+ insertData();
+
+ PlanExecutor executor = new PlanExecutor();
+ QueryPlan queryPlan =
+ (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp.d1");
+ QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT);
+ Assert.assertEquals(3, dataSet.getPaths().size());
+ int count = 0;
+ while (dataSet.hasNext()) {
+ RowRecord record = dataSet.next();
+ assertTrue(retSet.contains(record.toString()));
+ count++;
+ }
+
+ assertEquals(retSet.size(), count);
+
+ // test it from id table
+ assertEquals(
+ new TimeValuePair(113L, new TsDouble(4.0d)),
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.isp.d1"))
+ .getIdTable()
+ .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s1"))));
+ assertEquals(
+ new TimeValuePair(113L, new TsFloat(5.0f)),
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.isp.d1"))
+ .getIdTable()
+ .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s2"))));
+ assertEquals(
+ new TimeValuePair(113L, new TsLong(10003L)),
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.isp.d1"))
+ .getIdTable()
+ .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s3"))));
+ assertEquals(
+ new TimeValuePair(113L, new TsInt(103)),
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.isp.d1"))
+ .getIdTable()
+ .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s4"))));
+ assertEquals(
+ new TimeValuePair(113L, new TsBoolean(false)),
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.isp.d1"))
+ .getIdTable()
+ .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s5"))));
+ assertEquals(
+ new TimeValuePair(113L, new TsBinary(new Binary("hh3"))),
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.isp.d1"))
+ .getIdTable()
+ .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s6"))));
+ }
+
+ private void insertData() throws IllegalPathException, QueryProcessException {
+ long[] times = new long[] {110L, 111L, 112L, 113L};
+ List<Integer> dataTypes = new ArrayList<>();
+ dataTypes.add(TSDataType.DOUBLE.ordinal());
+ dataTypes.add(TSDataType.FLOAT.ordinal());
+ dataTypes.add(TSDataType.INT64.ordinal());
+ dataTypes.add(TSDataType.INT32.ordinal());
+ dataTypes.add(TSDataType.BOOLEAN.ordinal());
+ dataTypes.add(TSDataType.TEXT.ordinal());
+
+ Object[] columns = new Object[6];
+ columns[0] = new double[4];
+ columns[1] = new float[4];
+ columns[2] = new long[4];
+ columns[3] = new int[4];
+ columns[4] = new boolean[4];
+ columns[5] = new Binary[4];
+
+ for (int r = 0; r < 4; r++) {
+ ((double[]) columns[0])[r] = 1.0 + r;
+ ((float[]) columns[1])[r] = 2 + r;
+ ((long[]) columns[2])[r] = 10000 + r;
+ ((int[]) columns[3])[r] = 100 + r;
+ ((boolean[]) columns[4])[r] = false;
+ ((Binary[]) columns[5])[r] = new Binary("hh" + r);
+ }
+
+ InsertTabletPlan tabletPlan =
+ new InsertTabletPlan(
+ new PartialPath("root.isp.d1"),
+ new String[] {"s1", "s2", "s3", "s4", "s5", "s6"},
+ dataTypes);
+ tabletPlan.setTimes(times);
+ tabletPlan.setColumns(columns);
+ tabletPlan.setRowCount(times.length);
+
+ PlanExecutor executor = new PlanExecutor();
+ executor.insertTablet(tabletPlan);
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java
index 2da2b7d..0550e33 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/id_table/entry/DeviceIDTest.java
@@ -19,13 +19,14 @@
package org.apache.iotdb.db.metadata.id_table.entry;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.path.PartialPath;
+
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
public class DeviceIDTest {
@Test
public void deviceIDBuildTest() throws IllegalPathException {