You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/03/05 08:48:20 UTC
[incubator-iotdb] branch master updated: Cherry pick the changes of
the distributed version (#866)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 890eed6 Cherry pick the changes of the distributed version (#866)
890eed6 is described below
commit 890eed6af6791ccfe3cb1f473969b7140364b004
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Thu Mar 5 16:48:13 2020 +0800
Cherry pick the changes of the distributed version (#866)
* cherry pick the changes of the distributed version
Co-authored-by: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
---
.../resources/conf/iotdb-engine.properties | 2 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 141 +++++++-------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +
.../ServerConfigConsistent.java} | 14 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 88 ++++++++-
.../engine/storagegroup/StorageGroupProcessor.java | 91 +++++----
.../db/engine/storagegroup/TsFileResource.java | 35 +++-
.../iotdb/db/exception/StorageEngineException.java | 6 +-
.../exception/metadata/IllegalPathException.java | 4 +-
.../db/exception/metadata/MetadataException.java | 4 +
.../exception/metadata/PathNotExistException.java | 3 -
.../metadata/StorageGroupAlreadySetException.java | 3 +-
.../db/exception/query/QueryProcessException.java | 2 +-
.../runtime/StorageEngineFailureException.java | 4 +-
.../StorageGroupProcessorException.java | 2 +-
.../org/apache/iotdb/db/metadata/MManager.java | 69 +++++++
.../java/org/apache/iotdb/db/metadata/MTree.java | 61 ++++++
.../main/java/org/apache/iotdb/db/qp/Planner.java | 9 +-
.../apache/iotdb/db/qp/executor/IPlanExecutor.java | 7 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 68 +++----
.../iotdb/db/qp/logical/crud/FilterOperator.java | 10 +-
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 43 ++++-
.../iotdb/db/qp/physical/crud/BatchInsertPlan.java | 212 ++++++++++++++++-----
.../iotdb/db/qp/physical/crud/DeletePlan.java | 10 +
.../iotdb/db/qp/physical/crud/InsertPlan.java | 25 +++
.../apache/iotdb/db/qp/physical/sys/CountPlan.java | 3 +
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 66 +++++--
.../db/qp/physical/sys/SetStorageGroupPlan.java | 48 ++++-
.../db/qp/physical/sys/ShowChildPathsPlan.java | 11 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 22 ++-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 23 ++-
.../db/query/aggregation/AggregateResult.java | 7 +-
.../db/query/control/QueryResourceManager.java | 2 +-
.../db/query/dataset/AlignByDeviceDataSet.java | 2 +-
.../dataset/RawQueryDataSetWithValueFilter.java | 6 +-
.../groupby/GroupByWithValueFilterDataSet.java | 8 +-
.../db/query/executor/AggregationExecutor.java | 173 ++++++++++-------
.../iotdb/db/query/executor/QueryRouter.java | 11 +-
.../db/query/executor/RawDataQueryExecutor.java | 30 ++-
.../db/query/externalsort/ExternalSortJob.java | 3 +-
.../externalsort/ExternalSortJobScheduler.java | 19 +-
.../externalsort/SimpleExternalSortEngine.java | 18 +-
.../reader/series/SeriesReaderByTimestamp.java | 4 +
.../reader/universal/PriorityMergeReader.java | 2 +-
.../query/timegenerator/ServerTimeGenerator.java | 8 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 +-
.../apache/iotdb/db/service/MetricsService.java | 2 +-
.../org/apache/iotdb/db/service/ServiceType.java | 3 +-
.../org/apache/iotdb/db/service/StaticResps.java | 12 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 150 ++++++---------
.../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 2 +-
.../iotdb/db/tools/TsFileResourcePrinter.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 85 ++++++++-
.../java/org/apache/iotdb/db/utils/TestOnly.java | 4 +
.../writelog/recover/TsFileRecoverPerformer.java | 2 +-
.../apache/iotdb/db/engine/merge/MergeLogTest.java | 3 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 5 +-
.../iotdb/db/engine/merge/MergeTaskTest.java | 5 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 5 +-
.../engine/modification/DeletionFileNodeTest.java | 9 +-
.../db/engine/modification/DeletionQueryTest.java | 6 +-
.../storagegroup/FileNodeManagerBenchmark.java | 7 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 4 +-
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 2 +-
.../db/integration/IoTDBSequenceDataQueryIT.java | 5 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 2 +-
.../apache/iotdb/db/integration/IoTDBTtlIT.java | 1 -
.../iotdb/db/qp/plan/ConcatOptimizerTest.java | 3 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 6 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +-
.../apache/iotdb/tsfile/read/TimeValuePair.java | 1 +
.../apache/iotdb/tsfile/read/common/BatchData.java | 12 +-
.../apache/iotdb/tsfile/read/common/TimeRange.java | 4 +
.../iotdb/tsfile/read/filter/GroupByFilter.java | 8 +
.../iotdb/tsfile/read/filter/operator/In.java | 4 +
.../apache/iotdb/tsfile/utils/Murmur128Hash.java | 84 ++++----
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 7 +-
.../tsfile/write/schema/MeasurementSchema.java | 1 +
79 files changed, 1281 insertions(+), 559 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index d3e4cee..1e263ac 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -21,6 +21,8 @@
### Web Page Configuration
####################
+enable_metric_service=false
+
metrics_port=8181
query_cache_size_in_metric=200
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 1fc73b1..c8ad793 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -47,6 +47,9 @@ public class IoTDBConfig {
* Port which the metrics service listens to.
*/
private int metricsPort = 8181;
+
+ private boolean enableMetricService = false;
+
private String rpcAddress = "0.0.0.0";
/**
@@ -483,8 +486,6 @@ public class IoTDBConfig {
//wait for 60 second by default.
private int thriftServerAwaitTimeForStopService = 60;
- private boolean enableMetricsWebService = true;
-
private int queryCacheSizeInMetric = 200;
public IoTDBConfig() {
@@ -495,7 +496,7 @@ public class IoTDBConfig {
return memtableNumInEachStorageGroup;
}
- public void setMemtableNumInEachStorageGroup(int memtableNumInEachStorageGroup) {
+ void setMemtableNumInEachStorageGroup(int memtableNumInEachStorageGroup) {
this.memtableNumInEachStorageGroup = memtableNumInEachStorageGroup;
}
@@ -611,18 +612,26 @@ public class IoTDBConfig {
return dataDirs;
}
- void setDataDirs(String[] dataDirs) {
- this.dataDirs = dataDirs;
- }
-
public int getMetricsPort() {
return metricsPort;
}
- public void setMetricsPort(int metricsPort) {
+ void setMetricsPort(int metricsPort) {
this.metricsPort = metricsPort;
}
+ public boolean isEnableMetricService() {
+ return enableMetricService;
+ }
+
+ public void setEnableMetricService(boolean enableMetricService) {
+ this.enableMetricService = enableMetricService;
+ }
+
+ void setDataDirs(String[] dataDirs) {
+ this.dataDirs = dataDirs;
+ }
+
public String getRpcAddress() {
return rpcAddress;
}
@@ -643,7 +652,7 @@ public class IoTDBConfig {
return timestampPrecision;
}
- public void setTimestampPrecision(String timestampPrecision) {
+ void setTimestampPrecision(String timestampPrecision) {
this.timestampPrecision = timestampPrecision;
}
@@ -691,7 +700,7 @@ public class IoTDBConfig {
return syncDir;
}
- public void setSyncDir(String syncDir) {
+ void setSyncDir(String syncDir) {
this.syncDir = syncDir;
}
@@ -699,7 +708,7 @@ public class IoTDBConfig {
return queryDir;
}
- public void setQueryDir(String queryDir) {
+ void setQueryDir(String queryDir) {
this.queryDir = queryDir;
}
@@ -779,7 +788,7 @@ public class IoTDBConfig {
return rpcMaxConcurrentClientNum;
}
- public void setRpcMaxConcurrentClientNum(int rpcMaxConcurrentClientNum) {
+ void setRpcMaxConcurrentClientNum(int rpcMaxConcurrentClientNum) {
this.rpcMaxConcurrentClientNum = rpcMaxConcurrentClientNum;
}
@@ -823,7 +832,7 @@ public class IoTDBConfig {
this.syncServerPort = syncServerPort;
}
- public String getLanguageVersion() {
+ String getLanguageVersion() {
return languageVersion;
}
@@ -835,7 +844,7 @@ public class IoTDBConfig {
return baseDir;
}
- public void setBaseDir(String baseDir) {
+ void setBaseDir(String baseDir) {
this.baseDir = baseDir;
}
@@ -891,7 +900,7 @@ public class IoTDBConfig {
return mergeMemoryBudget;
}
- public void setMergeMemoryBudget(long mergeMemoryBudget) {
+ void setMergeMemoryBudget(long mergeMemoryBudget) {
this.mergeMemoryBudget = mergeMemoryBudget;
}
@@ -899,7 +908,7 @@ public class IoTDBConfig {
return mergeThreadNum;
}
- public void setMergeThreadNum(int mergeThreadNum) {
+ void setMergeThreadNum(int mergeThreadNum) {
this.mergeThreadNum = mergeThreadNum;
}
@@ -907,7 +916,7 @@ public class IoTDBConfig {
return continueMergeAfterReboot;
}
- public void setContinueMergeAfterReboot(boolean continueMergeAfterReboot) {
+ void setContinueMergeAfterReboot(boolean continueMergeAfterReboot) {
this.continueMergeAfterReboot = continueMergeAfterReboot;
}
@@ -915,7 +924,7 @@ public class IoTDBConfig {
return mergeIntervalSec;
}
- public void setMergeIntervalSec(long mergeIntervalSec) {
+ void setMergeIntervalSec(long mergeIntervalSec) {
this.mergeIntervalSec = mergeIntervalSec;
}
@@ -935,11 +944,11 @@ public class IoTDBConfig {
this.allocateMemoryForWrite = allocateMemoryForWrite;
}
- public long getAllocateMemoryForRead() {
+ long getAllocateMemoryForRead() {
return allocateMemoryForRead;
}
- public void setAllocateMemoryForRead(long allocateMemoryForRead) {
+ void setAllocateMemoryForRead(long allocateMemoryForRead) {
this.allocateMemoryForRead = allocateMemoryForRead;
}
@@ -947,7 +956,7 @@ public class IoTDBConfig {
return enableExternalSort;
}
- public void setEnableExternalSort(boolean enableExternalSort) {
+ void setEnableExternalSort(boolean enableExternalSort) {
this.enableExternalSort = enableExternalSort;
}
@@ -955,7 +964,7 @@ public class IoTDBConfig {
return externalSortThreshold;
}
- public void setExternalSortThreshold(int externalSortThreshold) {
+ void setExternalSortThreshold(int externalSortThreshold) {
this.externalSortThreshold = externalSortThreshold;
}
@@ -971,7 +980,7 @@ public class IoTDBConfig {
return performanceStatDisplayInterval;
}
- public void setPerformanceStatDisplayInterval(long performanceStatDisplayInterval) {
+ void setPerformanceStatDisplayInterval(long performanceStatDisplayInterval) {
this.performanceStatDisplayInterval = performanceStatDisplayInterval;
}
@@ -979,7 +988,7 @@ public class IoTDBConfig {
return performanceStatMemoryInKB;
}
- public void setPerformanceStatMemoryInKB(int performanceStatMemoryInKB) {
+ void setPerformanceStatMemoryInKB(int performanceStatMemoryInKB) {
this.performanceStatMemoryInKB = performanceStatMemoryInKB;
}
@@ -987,7 +996,7 @@ public class IoTDBConfig {
return forceFullMerge;
}
- public void setForceFullMerge(boolean forceFullMerge) {
+ void setForceFullMerge(boolean forceFullMerge) {
this.forceFullMerge = forceFullMerge;
}
@@ -1020,7 +1029,7 @@ public class IoTDBConfig {
return mergeChunkSubThreadNum;
}
- public void setMergeChunkSubThreadNum(int mergeChunkSubThreadNum) {
+ void setMergeChunkSubThreadNum(int mergeChunkSubThreadNum) {
this.mergeChunkSubThreadNum = mergeChunkSubThreadNum;
}
@@ -1028,7 +1037,7 @@ public class IoTDBConfig {
return mergeFileSelectionTimeBudget;
}
- public void setMergeFileSelectionTimeBudget(long mergeFileSelectionTimeBudget) {
+ void setMergeFileSelectionTimeBudget(long mergeFileSelectionTimeBudget) {
this.mergeFileSelectionTimeBudget = mergeFileSelectionTimeBudget;
}
@@ -1036,7 +1045,7 @@ public class IoTDBConfig {
return rpcThriftCompressionEnable;
}
- public void setRpcThriftCompressionEnable(boolean rpcThriftCompressionEnable) {
+ void setRpcThriftCompressionEnable(boolean rpcThriftCompressionEnable) {
this.rpcThriftCompressionEnable = rpcThriftCompressionEnable;
}
@@ -1052,7 +1061,7 @@ public class IoTDBConfig {
return allocateMemoryForFileMetaDataCache;
}
- public void setAllocateMemoryForFileMetaDataCache(long allocateMemoryForFileMetaDataCache) {
+ void setAllocateMemoryForFileMetaDataCache(long allocateMemoryForFileMetaDataCache) {
this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache;
}
@@ -1096,7 +1105,7 @@ public class IoTDBConfig {
this.watermarkBitString = watermarkBitString;
}
- public String getWatermarkMethod() {
+ String getWatermarkMethod() {
return this.watermarkMethod;
}
@@ -1116,7 +1125,7 @@ public class IoTDBConfig {
return Integer.parseInt(getWatermarkParamValue("embed_lsb_num", "5"));
}
- public String getWatermarkParamValue(String key, String defaultValue) {
+ private String getWatermarkParamValue(String key, String defaultValue) {
String res = getWatermarkParamValue(key);
if (res != null) {
return res;
@@ -1124,7 +1133,7 @@ public class IoTDBConfig {
return defaultValue;
}
- public String getWatermarkParamValue(String key) {
+ private String getWatermarkParamValue(String key) {
String pattern = key + "=(\\w*)";
Pattern r = Pattern.compile(pattern);
Matcher m = r.matcher(watermarkMethod);
@@ -1146,7 +1155,7 @@ public class IoTDBConfig {
return defaultStorageGroupLevel;
}
- public void setDefaultStorageGroupLevel(int defaultStorageGroupLevel) {
+ void setDefaultStorageGroupLevel(int defaultStorageGroupLevel) {
this.defaultStorageGroupLevel = defaultStorageGroupLevel;
}
@@ -1158,7 +1167,7 @@ public class IoTDBConfig {
this.defaultBooleanEncoding = defaultBooleanEncoding;
}
- public void setDefaultBooleanEncoding(String defaultBooleanEncoding) {
+ void setDefaultBooleanEncoding(String defaultBooleanEncoding) {
this.defaultBooleanEncoding = TSEncoding.valueOf(defaultBooleanEncoding);
}
@@ -1170,7 +1179,7 @@ public class IoTDBConfig {
this.defaultInt32Encoding = defaultInt32Encoding;
}
- public void setDefaultInt32Encoding(String defaultInt32Encoding) {
+ void setDefaultInt32Encoding(String defaultInt32Encoding) {
this.defaultInt32Encoding = TSEncoding.valueOf(defaultInt32Encoding);
}
@@ -1182,7 +1191,7 @@ public class IoTDBConfig {
this.defaultInt64Encoding = defaultInt64Encoding;
}
- public void setDefaultInt64Encoding(String defaultInt64Encoding) {
+ void setDefaultInt64Encoding(String defaultInt64Encoding) {
this.defaultInt64Encoding = TSEncoding.valueOf(defaultInt64Encoding);
}
@@ -1194,7 +1203,7 @@ public class IoTDBConfig {
this.defaultFloatEncoding = defaultFloatEncoding;
}
- public void setDefaultFloatEncoding(String defaultFloatEncoding) {
+ void setDefaultFloatEncoding(String defaultFloatEncoding) {
this.defaultFloatEncoding = TSEncoding.valueOf(defaultFloatEncoding);
}
@@ -1206,7 +1215,7 @@ public class IoTDBConfig {
this.defaultDoubleEncoding = defaultDoubleEncoding;
}
- public void setDefaultDoubleEncoding(String defaultDoubleEncoding) {
+ void setDefaultDoubleEncoding(String defaultDoubleEncoding) {
this.defaultDoubleEncoding = TSEncoding.valueOf(defaultDoubleEncoding);
}
@@ -1218,7 +1227,7 @@ public class IoTDBConfig {
this.defaultTextEncoding = defaultTextEncoding;
}
- public void setDefaultTextEncoding(String defaultTextEncoding) {
+ void setDefaultTextEncoding(String defaultTextEncoding) {
this.defaultTextEncoding = TSEncoding.valueOf(defaultTextEncoding);
}
@@ -1230,27 +1239,27 @@ public class IoTDBConfig {
this.systemFileStorageFs = FSType.valueOf(systemFileStorageFs);
}
- public FSType getTsFileStorageFs() {
+ FSType getTsFileStorageFs() {
return tsFileStorageFs;
}
- public void setTsFileStorageFs(String tsFileStorageFs) {
+ void setTsFileStorageFs(String tsFileStorageFs) {
this.tsFileStorageFs = FSType.valueOf(tsFileStorageFs);
}
- public String getCoreSitePath() {
+ String getCoreSitePath() {
return coreSitePath;
}
- public void setCoreSitePath(String coreSitePath) {
+ void setCoreSitePath(String coreSitePath) {
this.coreSitePath = coreSitePath;
}
- public String getHdfsSitePath() {
+ String getHdfsSitePath() {
return hdfsSitePath;
}
- public void setHdfsSitePath(String hdfsSitePath) {
+ void setHdfsSitePath(String hdfsSitePath) {
this.hdfsSitePath = hdfsSitePath;
}
@@ -1262,15 +1271,15 @@ public class IoTDBConfig {
return hdfsIp;
}
- public void setHdfsIp(String[] hdfsIp) {
+ void setHdfsIp(String[] hdfsIp) {
this.hdfsIp = String.join(",", hdfsIp);
}
- public String getHdfsPort() {
+ String getHdfsPort() {
return hdfsPort;
}
- public void setHdfsPort(String hdfsPort) {
+ void setHdfsPort(String hdfsPort) {
this.hdfsPort = hdfsPort;
}
@@ -1278,15 +1287,15 @@ public class IoTDBConfig {
return upgradeThreadNum;
}
- public void setUpgradeThreadNum(int upgradeThreadNum) {
+ void setUpgradeThreadNum(int upgradeThreadNum) {
this.upgradeThreadNum = upgradeThreadNum;
}
- public String getDfsNameServices() {
+ String getDfsNameServices() {
return dfsNameServices;
}
- public void setDfsNameServices(String dfsNameServices) {
+ void setDfsNameServices(String dfsNameServices) {
this.dfsNameServices = dfsNameServices;
}
@@ -1298,47 +1307,47 @@ public class IoTDBConfig {
return dfsHaNamenodes;
}
- public void setDfsHaNamenodes(String[] dfsHaNamenodes) {
+ void setDfsHaNamenodes(String[] dfsHaNamenodes) {
this.dfsHaNamenodes = String.join(",", dfsHaNamenodes);
}
- public boolean isDfsHaAutomaticFailoverEnabled() {
+ boolean isDfsHaAutomaticFailoverEnabled() {
return dfsHaAutomaticFailoverEnabled;
}
- public void setDfsHaAutomaticFailoverEnabled(boolean dfsHaAutomaticFailoverEnabled) {
+ void setDfsHaAutomaticFailoverEnabled(boolean dfsHaAutomaticFailoverEnabled) {
this.dfsHaAutomaticFailoverEnabled = dfsHaAutomaticFailoverEnabled;
}
- public String getDfsClientFailoverProxyProvider() {
+ String getDfsClientFailoverProxyProvider() {
return dfsClientFailoverProxyProvider;
}
- public void setDfsClientFailoverProxyProvider(String dfsClientFailoverProxyProvider) {
+ void setDfsClientFailoverProxyProvider(String dfsClientFailoverProxyProvider) {
this.dfsClientFailoverProxyProvider = dfsClientFailoverProxyProvider;
}
- public boolean isUseKerberos() {
+ boolean isUseKerberos() {
return useKerberos;
}
- public void setUseKerberos(boolean useKerberos) {
+ void setUseKerberos(boolean useKerberos) {
this.useKerberos = useKerberos;
}
- public String getKerberosKeytabFilePath() {
+ String getKerberosKeytabFilePath() {
return kerberosKeytabFilePath;
}
- public void setKerberosKeytabFilePath(String kerberosKeytabFilePath) {
+ void setKerberosKeytabFilePath(String kerberosKeytabFilePath) {
this.kerberosKeytabFilePath = kerberosKeytabFilePath;
}
- public String getKerberosPrincipal() {
+ String getKerberosPrincipal() {
return kerberosPrincipal;
}
- public void setKerberosPrincipal(String kerberosPrincipal) {
+ void setKerberosPrincipal(String kerberosPrincipal) {
this.kerberosPrincipal = kerberosPrincipal;
}
@@ -1358,14 +1367,6 @@ public class IoTDBConfig {
this.thriftServerAwaitTimeForStopService = thriftServerAwaitTimeForStopService;
}
- public boolean isEnableMetricsWebService() {
- return enableMetricsWebService;
- }
-
- public void setEnableMetricsWebService(boolean enableMetricsWebService) {
- this.enableMetricsWebService = enableMetricsWebService;
- }
-
public int getQueryCacheSizeInMetric() {
return queryCacheSizeInMetric;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index aa318f6..d0131dc 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -114,6 +114,9 @@ public class IoTDBDescriptor {
+ " use default value");
}
+ conf.setEnableMetricService(Boolean.parseBoolean(properties
+ .getProperty("enable_metric_service", Boolean.toString(conf.isEnableMetricService()))));
+
conf.setMetricsPort(Integer.parseInt(properties.getProperty("metrics_port",
Integer.toString(conf.getMetricsPort()))));
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java b/server/src/main/java/org/apache/iotdb/db/conf/ServerConfigConsistent.java
similarity index 76%
copy from server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java
copy to server/src/main/java/org/apache/iotdb/db/conf/ServerConfigConsistent.java
index d4058ba..18ad0a4 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/ServerConfigConsistent.java
@@ -15,17 +15,23 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
*/
-package org.apache.iotdb.db.utils;
+package org.apache.iotdb.db.conf;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-@Target({ElementType.METHOD, ElementType.CONSTRUCTOR})
+/**
+ * Configurations using this annotation should be the same across all nodes in a cluster.
+ */
+@Target({ElementType.FIELD})
@Retention(RetentionPolicy.SOURCE)
-public @interface TestOnly {
+public @interface ServerConfigConsistent {
+ //TODO#IOTDB-436: the restarted server should check the
+ // configuration consistency as before.
+
}
+
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index cf99279..18485dc 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -23,8 +23,10 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.ConcurrentModificationException;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
@@ -38,6 +40,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.ServerConfigConsistent;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy;
import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy.DirectFlushPolicy;
@@ -91,15 +94,46 @@ public class StorageEngine implements IService {
private static final ExecutorService recoveryThreadPool = IoTDBThreadPoolFactory
.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), "Recovery-Thread-Pool");
- private static final StorageEngine INSTANCE = new StorageEngine();
+ static class InstanceHolder {
+
+ private InstanceHolder() {
+ // forbidding instantiation
+ }
+
+ private static final StorageEngine INSTANCE = new StorageEngine();
+ }
public static StorageEngine getInstance() {
- return INSTANCE;
+ return InstanceHolder.INSTANCE;
}
private ScheduledExecutorService ttlCheckThread;
private TsFileFlushPolicy fileFlushPolicy = new DirectFlushPolicy();
+ /**
+ * Time range for dividing storage group, the time unit is the same with IoTDB's TimestampPrecision
+ */
+ @ServerConfigConsistent
+ static long timePartitionInterval;
+ static {
+ // build time Interval to divide time partition
+ String timePrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ switch (timePrecision) {
+ case "ns":
+ timePartitionInterval = IoTDBDescriptor.getInstance().
+ getConfig().getPartitionInterval() * 1000_000_000L;
+ break;
+ case "us":
+ timePartitionInterval = IoTDBDescriptor.getInstance().
+ getConfig().getPartitionInterval() * 1000_000L;
+ break;
+ default:
+ timePartitionInterval = IoTDBDescriptor.getInstance().
+ getConfig().getPartitionInterval() * 1000;
+ break;
+ }
+ }
+
private StorageEngine() {
logger = LoggerFactory.getLogger(StorageEngine.class);
systemDir = FilePathUtils.regularizePath(config.getSystemDir()) + "storage_groups";
@@ -109,7 +143,6 @@ public class StorageEngine implements IService {
} catch (IOException e) {
throw new StorageEngineFailureException(e);
}
-
// recover upgrade process
UpgradeUtils.recoverUpgrade();
/*
@@ -160,14 +193,16 @@ public class StorageEngine implements IService {
@Override
public void stop() {
syncCloseAllProcessor();
- ttlCheckThread.shutdownNow();
+ if (ttlCheckThread != null) {
+ ttlCheckThread.shutdownNow();
+ try {
+ ttlCheckThread.awaitTermination(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ logger.warn("TTL check thread still doesn't exit after 30s");
+ }
+ }
recoveryThreadPool.shutdownNow();
this.reset();
- try {
- ttlCheckThread.awaitTermination(30, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- logger.warn("TTL check thread still doesn't exit after 30s");
- }
}
@Override
@@ -176,7 +211,7 @@ public class StorageEngine implements IService {
}
public StorageGroupProcessor getProcessor(String path) throws StorageEngineException {
- String storageGroupName = "";
+ String storageGroupName;
try {
storageGroupName = MManager.getInstance().getStorageGroupName(path);
StorageGroupProcessor processor;
@@ -490,4 +525,37 @@ public class StorageEngine implements IService {
return file.getParentFile().getParentFile().getName();
}
+ /**
+ *
+ * @return TsFiles (seq or unseq) grouped by their storage group.
+ */
+ public Map<String, List<TsFileResource>> getAllClosedStorageGroupTsFile() {
+ Map<String, List<TsFileResource>> ret = new HashMap<>();
+ for (Entry<String, StorageGroupProcessor> entry : processorMap
+ .entrySet()) {
+ ret.computeIfAbsent(entry.getKey(), sg -> new ArrayList<>()).addAll(entry.getValue().getSequenceFileTreeSet());
+ ret.get(entry.getKey()).addAll(entry.getValue().getUnSequenceFileList());
+ ret.get(entry.getKey()).removeIf(file -> !file.isClosed());
+ }
+ return ret;
+ }
+
+ public void setFileFlushPolicy(TsFileFlushPolicy fileFlushPolicy) {
+ this.fileFlushPolicy = fileFlushPolicy;
+ }
+
+ public boolean isFileAlreadyExist(TsFileResource tsFileResource, String storageGroup) {
+ // TODO-Cluster#350: integrate with time partitioning
+ StorageGroupProcessor processor = processorMap.get(storageGroup);
+ return processor != null && processor.isFileAlreadyExist(tsFileResource);
+ }
+
+ public static long getTimePartitionInterval() {
+ return timePartitionInterval;
+ }
+
+ public static long fromTimeToTimePartition(long time) {
+
+ return time / timePartitionInterval;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 210afa6..e220f7c 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -44,6 +44,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
@@ -77,7 +78,6 @@ import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryFileManager;
import org.apache.iotdb.db.utils.CopyOnReadLinkedList;
-import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.UpgradeUtils;
import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -145,10 +145,7 @@ public class StorageGroupProcessor {
* time partition id in the storage group -> tsFileProcessor for this time partition
*/
private final TreeMap<Long, TsFileProcessor> workUnsequenceTsFileProcessors = new TreeMap<>();
- /**
- * Time range for dividing storage group, unit is second
- */
- private long partitionIntervalForStorageGroup;
+
/**
* the schema of time series that belong this storage group
*/
@@ -240,23 +237,6 @@ public class StorageGroupProcessor {
storageGroupSysDir.getPath());
}
- // build time Interval to divide time partition
- String timePrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
- switch (timePrecision) {
- case "ns":
- partitionIntervalForStorageGroup = IoTDBDescriptor.getInstance().
- getConfig().getPartitionInterval() * 1000_000_000L;
- break;
- case "us":
- partitionIntervalForStorageGroup = IoTDBDescriptor.getInstance().
- getConfig().getPartitionInterval() * 1000_000L;
- break;
- default:
- partitionIntervalForStorageGroup = IoTDBDescriptor.getInstance().
- getConfig().getPartitionInterval() * 1000;
- break;
- }
-
recover();
}
@@ -333,7 +313,7 @@ public class StorageGroupProcessor {
// just find any time of device
Iterator<Long> iterator = startTimeMap.values().iterator();
if (iterator.hasNext()) {
- return fromTimeToTimePartition(iterator.next());
+ return StorageEngine.fromTimeToTimePartition(iterator.next());
}
return -1;
@@ -506,7 +486,7 @@ public class StorageGroupProcessor {
writeLock();
try {
// init map
- long timePartitionId = fromTimeToTimePartition(insertPlan.getTime());
+ long timePartitionId = StorageEngine.fromTimeToTimePartition(insertPlan.getTime());
latestTimeForEachDevice.computeIfAbsent(timePartitionId, l -> new HashMap<>())
.putIfAbsent(insertPlan.getDeviceId(), Long.MIN_VALUE);
partitionLatestFlushedTimeForEachDevice.computeIfAbsent(timePartitionId, id -> new HashMap<>())
@@ -547,7 +527,7 @@ public class StorageGroupProcessor {
// before is first start point
int before = loc;
// before time partition
- long beforeTimePartition = fromTimeToTimePartition(batchInsertPlan.getTimes()[before]);
+ long beforeTimePartition = StorageEngine.fromTimeToTimePartition(batchInsertPlan.getTimes()[before]);
// init map
long lastFlushTime = partitionLatestFlushedTimeForEachDevice.
computeIfAbsent(beforeTimePartition, id -> new HashMap<>()).
@@ -556,7 +536,7 @@ public class StorageGroupProcessor {
boolean isSequence = false;
while (loc < batchInsertPlan.getRowCount()) {
long time = batchInsertPlan.getTimes()[loc];
- long curTimePartition = fromTimeToTimePartition(time);
+ long curTimePartition = StorageEngine.fromTimeToTimePartition(time);
results[loc] = TSStatusCode.SUCCESS_STATUS.getStatusCode();
// start next partition
if (curTimePartition != beforeTimePartition) {
@@ -675,7 +655,7 @@ public class StorageGroupProcessor {
throws QueryProcessException {
TsFileProcessor tsFileProcessor;
boolean result;
- long timePartitionId = fromTimeToTimePartition(insertPlan.getTime());
+ long timePartitionId = StorageEngine.fromTimeToTimePartition(insertPlan.getTime());
tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, sequence);
@@ -798,10 +778,6 @@ public class StorageGroupProcessor {
}
- private long fromTimeToTimePartition(long time) {
-
- return time / partitionIntervalForStorageGroup;
- }
private TsFileProcessor createTsFileProcessor(boolean sequence, long timePartitionId)
throws IOException, DiskSpaceInsufficientException {
@@ -847,6 +823,7 @@ public class StorageGroupProcessor {
}
private String getNewTsFileName(long time, long version, int mergeCnt) {
+ allDirectFileVersions.add(version);
return time + IoTDBConstant.TSFILE_NAME_SEPARATOR + version
+ IoTDBConstant.TSFILE_NAME_SEPARATOR + mergeCnt + TSFILE_SUFFIX;
}
@@ -1229,7 +1206,7 @@ public class StorageGroupProcessor {
}
// time partition to divide storage group
- long timePartitionId = fromTimeToTimePartition(timestamp);
+ long timePartitionId = StorageEngine.fromTimeToTimePartition(timestamp);
// write log
if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
DeletePlan deletionPlan = new DeletePlan(timestamp, new Path(deviceId, measurementId));
@@ -1684,6 +1661,40 @@ public class StorageGroupProcessor {
}
/**
+ * If the historical versions of a file is a sub-set of the given file's, remove it to reduce
+ * unnecessary merge. Only used when the file sender and the receiver share the same file
+ * close policy.
+ * @param resource
+ */
+ public void removeFullyOverlapFiles(TsFileResource resource) {
+ writeLock();
+ closeQueryLock.writeLock().lock();
+ try {
+ Iterator<TsFileResource> iterator = sequenceFileTreeSet.iterator();
+ removeFullyOverlapFiles(resource, iterator);
+
+ iterator = unSequenceFileList.iterator();
+ removeFullyOverlapFiles(resource, iterator);
+ } finally {
+ closeQueryLock.writeLock().unlock();
+ writeUnlock();
+ }
+ }
+
+ private void removeFullyOverlapFiles(TsFileResource resource, Iterator<TsFileResource> iterator) {
+ while (iterator.hasNext()) {
+ TsFileResource seqFile = iterator.next();
+ if (resource.getHistoricalVersions().containsAll(seqFile.getHistoricalVersions())
+ && !resource.getHistoricalVersions().equals(seqFile.getHistoricalVersions())
+ && seqFile.getWriteQueryLock().writeLock().tryLock()) {
+ iterator.remove();
+ seqFile.remove();
+ seqFile.getWriteQueryLock().writeLock().unlock();
+ }
+ }
+ }
+
+ /**
* Get an appropriate filename to ensure the order between files. The tsfile is named after
* ({systemTime}-{versionNum}-{mergeNum}.tsfile).
* <p>
@@ -1747,7 +1758,7 @@ public class StorageGroupProcessor {
for (Entry<String, Long> entry : newTsFileResource.getEndTimeMap().entrySet()) {
String device = entry.getKey();
long endTime = newTsFileResource.getEndTimeMap().get(device);
- long timePartitionId = fromTimeToTimePartition(endTime);
+ long timePartitionId = StorageEngine.fromTimeToTimePartition(endTime);
if (!latestTimeForEachDevice.computeIfAbsent(timePartitionId, id -> new HashMap<>())
.containsKey(device)
|| latestTimeForEachDevice.get(timePartitionId).get(device) < endTime) {
@@ -1760,7 +1771,7 @@ public class StorageGroupProcessor {
if (!latestFlushTimeForPartition.containsKey(device)
|| latestFlushTimeForPartition.get(device) < endTime) {
partitionLatestFlushedTimeForEachDevice
- .computeIfAbsent(timePartitionId, id -> new HashMap<String, Long>())
+ .computeIfAbsent(timePartitionId, id -> new HashMap<>())
.put(device, endTime);
}
if (!globalLatestFlushedTimeForEachDevice.containsKey(device)
@@ -1781,7 +1792,7 @@ public class StorageGroupProcessor {
TsFileResource tsFileResource)
throws TsFileProcessorException, DiskSpaceInsufficientException {
File targetFile;
- long timeRangeId = fromTimeToTimePartition(
+ long timeRangeId = StorageEngine.fromTimeToTimePartition(
tsFileResource.getStartTimeMap().entrySet().iterator().next().getValue());
switch (type) {
case LOAD_UNSEQUENCE:
@@ -1961,12 +1972,10 @@ public class StorageGroupProcessor {
checkFilesTTL();
}
- @TestOnly
public List<TsFileResource> getSequenceFileTreeSet() {
return new ArrayList<>(sequenceFileTreeSet);
}
- @TestOnly
public List<TsFileResource> getUnSequenceFileList() {
return unSequenceFileList;
}
@@ -1981,6 +1990,14 @@ public class StorageGroupProcessor {
void call(TsFileProcessor caller) throws TsFileProcessorException, IOException;
}
+ public String getStorageGroupName() {
+ return storageGroupName;
+ }
+
+ public boolean isFileAlreadyExist(TsFileResource tsFileResource) {
+ return allDirectFileVersions.containsAll(tsFileResource.getHistoricalVersions());
+ }
+
@FunctionalInterface
public interface UpdateEndTimeCallBack {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index e9356db..58ecd56 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -60,12 +60,12 @@ public class TsFileResource {
/**
* device -> start time
*/
- private Map<String, Long> startTimeMap;
+ protected Map<String, Long> startTimeMap;
/**
* device -> end time. It is null if it's an unsealed sequence tsfile
*/
- private Map<String, Long> endTimeMap;
+ protected Map<String, Long> endTimeMap;
public TsFileProcessor getProcessor() {
return processor;
@@ -101,6 +101,25 @@ public class TsFileResource {
private FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+ public TsFileResource() {
+ }
+
+ public TsFileResource(TsFileResource other) {
+ this.file = other.file;
+ this.startTimeMap = other.startTimeMap;
+ this.endTimeMap = other.endTimeMap;
+ this.processor = other.processor;
+ this.modFile = other.modFile;
+ this.closed = other.closed;
+ this.deleted = other.deleted;
+ this.isMerging = other.isMerging;
+ this.chunkMetaDataList = other.chunkMetaDataList;
+ this.readOnlyMemChunk = other.readOnlyMemChunk;
+ this.writeQueryLock = other.writeQueryLock;
+ this.fsFactory = other.fsFactory;
+ this.historicalVersions = other.historicalVersions;
+ }
+
/**
* for sealed TsFile, call setClosed to close TsFileResource
*/
@@ -162,7 +181,7 @@ public class TsFileResource {
fsFactory.moveFile(src, dest);
}
- public void deSerialize() throws IOException {
+ public void deserialize() throws IOException {
try (InputStream inputStream = fsFactory.getBufferedInputStream(
file + RESOURCE_SUFFIX)) {
int size = ReadWriteIOUtils.readInt(inputStream);
@@ -363,11 +382,19 @@ public class TsFileResource {
return false;
}
+ protected void setStartTimeMap(Map<String, Long> startTimeMap) {
+ this.startTimeMap = startTimeMap;
+ }
+
+ protected void setEndTimeMap(Map<String, Long> endTimeMap) {
+ this.endTimeMap = endTimeMap;
+ }
+
/**
* set a file flag indicating that the file is being closed, so during recovery we could know we
* should close the file.
*/
- public void setCloseFlag() {
+ void setCloseFlag() {
try {
new File(file.getAbsoluteFile() + CLOSING_SUFFIX).createNewFile();
} catch (IOException e) {
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/StorageEngineException.java b/server/src/main/java/org/apache/iotdb/db/exception/StorageEngineException.java
index 7944bd4..bd03a29 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/StorageEngineException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/StorageEngineException.java
@@ -24,6 +24,10 @@ public class StorageEngineException extends IoTDBException {
private static final long serialVersionUID = 9001649171768311032L;
+ public StorageEngineException(Throwable cause) {
+ super(cause, TSStatusCode.STORAGE_ENGINE_ERROR.getStatusCode());
+ }
+
public StorageEngineException(String message) {
super(message, TSStatusCode.STORAGE_ENGINE_ERROR.getStatusCode());
}
@@ -33,6 +37,6 @@ public class StorageEngineException extends IoTDBException {
}
public StorageEngineException(IoTDBException e) {
- super(e.getMessage(), e.getErrorCode());
+ super(e, e.getErrorCode());
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/IllegalPathException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/IllegalPathException.java
index 8f9ee27..da40603 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/metadata/IllegalPathException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/IllegalPathException.java
@@ -15,7 +15,6 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
*/
package org.apache.iotdb.db.exception.metadata;
@@ -23,11 +22,10 @@ package org.apache.iotdb.db.exception.metadata;
import org.apache.iotdb.rpc.TSStatusCode;
public class IllegalPathException extends MetadataException {
-
private static final long serialVersionUID = 2693272249167539978L;
public IllegalPathException(String path) {
super(String.format("%s is not a legal path", path));
errorCode = TSStatusCode.PATH_ILLEGAL.getStatusCode();
}
-}
\ No newline at end of file
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/MetadataException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/MetadataException.java
index dfca520..5d8921c 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/metadata/MetadataException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/MetadataException.java
@@ -29,6 +29,10 @@ public class MetadataException extends IoTDBException {
private static final long serialVersionUID = 3415275599091623570L;
+ public MetadataException(Throwable cause) {
+ super(cause, TSStatusCode.METADATA_ERROR.getStatusCode());
+ }
+
public MetadataException(Throwable cause, int errorCode) {
super(cause, TSStatusCode.METADATA_ERROR.getStatusCode());
this.errorCode = errorCode;
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/PathNotExistException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/PathNotExistException.java
index fe8e476..7d86ade 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/metadata/PathNotExistException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/PathNotExistException.java
@@ -15,15 +15,12 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
*/
package org.apache.iotdb.db.exception.metadata;
public class PathNotExistException extends MetadataException {
- private static final long serialVersionUID = 2693272249167539978L;
-
public PathNotExistException(String path) {
super(String.format("Path [%s] does not exist", path));
}
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/StorageGroupAlreadySetException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/StorageGroupAlreadySetException.java
index c43884a..b196309 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/metadata/StorageGroupAlreadySetException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/StorageGroupAlreadySetException.java
@@ -15,7 +15,6 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
*/
package org.apache.iotdb.db.exception.metadata;
@@ -27,4 +26,4 @@ public class StorageGroupAlreadySetException extends MetadataException {
public StorageGroupAlreadySetException(String path) {
super(String.format("%s has already been set to storage group", path));
}
-}
\ No newline at end of file
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java b/server/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java
index 9372d15..f8c007c 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/query/QueryProcessException.java
@@ -35,6 +35,6 @@ public class QueryProcessException extends IoTDBException {
}
public QueryProcessException(IoTDBException e) {
- super(e.getMessage(), e.getErrorCode());
+ super(e, e.getErrorCode());
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/runtime/StorageEngineFailureException.java b/server/src/main/java/org/apache/iotdb/db/exception/runtime/StorageEngineFailureException.java
index 934ece6..aab283c 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/runtime/StorageEngineFailureException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/runtime/StorageEngineFailureException.java
@@ -24,10 +24,10 @@ public class StorageEngineFailureException extends RuntimeException {
private static final long serialVersionUID = -1197701024139022020L;
public StorageEngineFailureException(Throwable e) {
- super("Create system directory failed! " + e.getMessage());
+ super("Create system directory failed! ", e);
}
public StorageEngineFailureException(String message, Throwable e) {
- super(message + e.getMessage());
+ super(message, e);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/storageGroup/StorageGroupProcessorException.java b/server/src/main/java/org/apache/iotdb/db/exception/storageGroup/StorageGroupProcessorException.java
index 249be85..9bae99d 100644
--- a/server/src/main/java/org/apache/iotdb/db/exception/storageGroup/StorageGroupProcessorException.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/storageGroup/StorageGroupProcessorException.java
@@ -26,7 +26,7 @@ public class StorageGroupProcessorException extends IoTDBException {
private static final long serialVersionUID = 7373978140952977661L;
public StorageGroupProcessorException(Exception exception) {
- super(exception.getMessage(), TSStatusCode.STORAGE_GROUP_PROCESSOR_ERROR.getStatusCode());
+ super(exception, TSStatusCode.STORAGE_GROUP_PROCESSOR_ERROR.getStatusCode());
}
public StorageGroupProcessorException(String message) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0ef29eb..00b21cc 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -24,9 +24,12 @@ import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -39,9 +42,11 @@ import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.exception.ConfigAdjusterException;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
+import org.apache.iotdb.db.metadata.mnode.LeafMNode;
import org.apache.iotdb.db.metadata.mnode.MNode;
import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode;
import org.apache.iotdb.db.monitor.MonitorConstants;
@@ -841,4 +846,68 @@ public class MManager {
lock.readLock().unlock();
}
}
+
+ public void collectSeries(MNode startingNode, Collection<MeasurementSchema> timeseriesSchemas) {
+ Deque<MNode> nodeDeque = new ArrayDeque<>();
+ nodeDeque.addLast(startingNode);
+ while (!nodeDeque.isEmpty()) {
+ MNode node = nodeDeque.removeFirst();
+ if (node instanceof LeafMNode) {
+ MeasurementSchema nodeSchema = node.getSchema();
+ timeseriesSchemas.add(new MeasurementSchema(node.getFullPath(),
+ nodeSchema.getType(), nodeSchema.getEncodingType(), nodeSchema.getCompressor()));
+ } else if (!node.getChildren().isEmpty()) {
+ nodeDeque.addAll(node.getChildren().values());
+ }
+ }
+ }
+
+ public void collectSeries(String startingPath, List<MeasurementSchema> timeseriesSchemas) {
+ MNode mNode;
+ try {
+ mNode = getNodeByPath(startingPath);
+ } catch (MetadataException e) {
+ return;
+ }
+ collectSeries(mNode, timeseriesSchemas);
+ }
+
+ /**
+ * For a path, infer all storage groups it may belong to.
+ * The path can have wildcards.
+ *
+ * Consider the path into two parts: (1) the sub path which can not contain a storage group name and
+ * (2) the sub path which is substring that begin after the storage group name.
+ *
+ * (1) Suppose the part of the path can not contain a storage group name (e.g.,
+ * "root".contains("root.sg") == false), then:
+ * If the wildcard is not at the tail, then for each wildcard, only one level will be inferred
+ * and the wildcard will be removed.
+ * If the wildcard is at the tail, then the inference will go on until the storage groups are found
+ * and the wildcard will be kept.
+ * (2) Suppose the part of the path is a substring that begin after the storage group name. (e.g.,
+ * For "root.*.sg1.a.*.b.*" and "root.x.sg1" is a storage group, then this part is "a.*.b.*").
+ * For this part, keep what it is.
+ *
+ * Assuming we have three SGs: root.group1, root.group2, root.area1.group3
+ * Eg1:
+ * for input "root.*", returns ("root.group1", "root.group1.*"), ("root.group2", "root.group2.*")
+ * ("root.area1.group3", "root.area1.group3.*")
+ * Eg2:
+ * for input "root.*.s1", returns ("root.group1", "root.group1.s1"), ("root.group2", "root.group2.s1")
+ *
+ * Eg3:
+ * for input "root.area1.*", returns ("root.area1.group3", "root.area1.group3.*")
+ *
+ * @param path can be a prefix or a full path.
+ * @return StorageGroupName-FullPath pairs
+ */
+ public Map<String, String> determineStorageGroup(String path) throws IllegalPathException {
+ lock.readLock().lock();
+ try {
+ return mtree.determineStorageGroup(path);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 9a07c20..87b6afd 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
@@ -231,6 +232,7 @@ public class MTree implements Serializable {
*
* @param path Format: root.node(.node)+
*/
+
String deleteTimeseriesAndReturnEmptyStorageGroup(String path) throws MetadataException {
MNode curNode = getNodeByPath(path);
if (!(curNode instanceof LeafMNode)) {
@@ -755,4 +757,63 @@ public class MTree implements Serializable {
}
return res;
}
+
+ Map<String, String> determineStorageGroup(String path) throws IllegalPathException {
+ Map<String, String> paths = new HashMap<>();
+ String[] nodes = MetaUtils.getNodeNames(path);
+ if (nodes.length == 0 || !nodes[0].equals(root.getName())) {
+ throw new IllegalPathException(path);
+ }
+
+ Deque<MNode> nodeStack = new ArrayDeque<>();
+ Deque<Integer> depthStack = new ArrayDeque<>();
+ if (!root.getChildren().isEmpty()) {
+ nodeStack.push(root);
+ depthStack.push(0);
+ }
+
+ while (!nodeStack.isEmpty()) {
+ MNode mNode = nodeStack.removeFirst();
+ int depth = depthStack.removeFirst();
+
+ determineStorageGroup(depth + 1, nodes, mNode, paths, nodeStack, depthStack);
+ }
+ return paths;
+ }
+
+ /**
+ * Try determining the storage group using the children of a mNode. If one child is a storage
+ * group node, put a storageGroupName-fullPath pair into paths. Otherwise put the children that
+ * match the path into the queue and discard other children.
+ */
+ private void determineStorageGroup(int depth, String[] nodes, MNode mNode,
+ Map<String, String> paths, Deque<MNode> nodeStack, Deque<Integer> depthStack) {
+ String currNode = depth >= nodes.length ? PATH_WILDCARD : nodes[depth];
+ for (Entry<String, MNode> entry : mNode.getChildren().entrySet()) {
+ if (!currNode.equals(PATH_WILDCARD) && !currNode.equals(entry.getKey())) {
+ continue;
+ }
+ // this child is desired
+ MNode child = entry.getValue();
+ if (child instanceof StorageGroupMNode) {
+ // we have found one storage group, record it
+ String sgName = child.getFullPath();
+ // concat the remaining path with the storage group name
+ StringBuilder pathWithKnownSG = new StringBuilder(sgName);
+ for (int i = depth + 1; i < nodes.length; i++) {
+ pathWithKnownSG.append(IoTDBConstant.PATH_SEPARATOR).append(nodes[i]);
+ }
+ if (depth >= nodes.length - 1 && currNode.equals(PATH_WILDCARD)) {
+ // the we find the sg at the last node and the last node is a wildcard (find "root
+ // .group1", for "root.*"), also append the wildcard (to make "root.group1.*")
+ pathWithKnownSG.append(IoTDBConstant.PATH_SEPARATOR).append(PATH_WILDCARD);
+ }
+ paths.put(sgName, pathWithKnownSG.toString());
+ } else if (!child.getChildren().isEmpty()) {
+ // push it back so we can traver its children later
+ nodeStack.push(child);
+ depthStack.push(depth);
+ }
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index 19a4d69..bd1089f 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -41,7 +41,7 @@ import org.apache.iotdb.db.utils.TestOnly;
*/
public class Planner {
- private ParseDriver parseDriver;
+ protected ParseDriver parseDriver;
public Planner() {
this.parseDriver = new ParseDriver();
@@ -70,7 +70,7 @@ public class Planner {
* @return optimized logical operator
* @throws LogicalOptimizeException exception in logical optimizing
*/
- private Operator logicalOptimize(Operator operator)
+ protected Operator logicalOptimize(Operator operator)
throws LogicalOperatorException {
switch (operator.getType()) {
case AUTHOR:
@@ -111,7 +111,7 @@ public class Planner {
*/
private SFWOperator optimizeSFWOperator(SFWOperator root)
throws LogicalOperatorException {
- ConcatPathOptimizer concatPathOptimizer = new ConcatPathOptimizer();
+ ConcatPathOptimizer concatPathOptimizer = getConcatPathOptimizer();
root = (SFWOperator) concatPathOptimizer.transform(root);
FilterOperator filter = root.getFilterOperator();
if (filter == null) {
@@ -127,4 +127,7 @@ public class Planner {
return root;
}
+ protected ConcatPathOptimizer getConcatPathOptimizer() {
+ return new ConcatPathOptimizer();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/IPlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/IPlanExecutor.java
index ec25529..b2466db 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/IPlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/IPlanExecutor.java
@@ -20,11 +20,13 @@ package org.apache.iotdb.db.qp.executor;
import java.io.IOException;
import java.sql.SQLException;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
-import org.apache.iotdb.db.qp.physical.crud.*;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
+import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -89,5 +91,4 @@ public interface IPlanExecutor {
* @return result of each row
*/
Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws QueryProcessException;
-
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 56047dc..d0c1251 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -34,6 +34,7 @@ import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TIMESERIES_ENCODING;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TTL;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_USER;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_VALUE;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
import java.io.File;
@@ -62,7 +63,7 @@ import org.apache.iotdb.db.engine.flush.pool.FlushTaskPoolManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
+import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.mnode.InternalMNode;
@@ -128,9 +129,7 @@ import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
public class PlanExecutor implements IPlanExecutor {
// for data query
- private IQueryRouter queryRouter;
- // for insert
- private StorageEngine storageEngine;
+ protected IQueryRouter queryRouter;
// for system schema
private MManager mManager;
// for administration
@@ -138,7 +137,6 @@ public class PlanExecutor implements IPlanExecutor {
public PlanExecutor() throws QueryProcessException {
queryRouter = new QueryRouter();
- storageEngine = StorageEngine.getInstance();
mManager = MManager.getInstance();
try {
authorizer = LocalFileAuthorizer.getInstance();
@@ -223,7 +221,7 @@ public class PlanExecutor implements IPlanExecutor {
}
}
- private QueryDataSet processDataQuery(QueryPlan queryPlan, QueryContext context)
+ protected QueryDataSet processDataQuery(QueryPlan queryPlan, QueryContext context)
throws StorageEngineException, QueryFilterOptimizationException, QueryProcessException,
IOException {
QueryDataSet queryDataSet;
@@ -319,7 +317,7 @@ public class PlanExecutor implements IPlanExecutor {
return MManager.getInstance().getAllTimeseriesName(path);
}
- private List<String> getNodesList(String schemaPattern, int level) throws MetadataException {
+ protected List<String> getNodesList(String schemaPattern, int level) throws MetadataException {
return MManager.getInstance().getNodesList(schemaPattern, level);
}
@@ -340,7 +338,7 @@ public class PlanExecutor implements IPlanExecutor {
throws MetadataException {
ListDataSet listDataSet = new ListDataSet(Collections.singletonList(new Path(COLUMN_DEVICES)),
Collections.singletonList(TSDataType.TEXT));
- Set<String> devices = MManager.getInstance().getDevices(showDevicesPlan.getPath().toString());
+ Set<String> devices = getDevices(showDevicesPlan.getPath().toString());
for (String s : devices) {
RowRecord record = new RowRecord(0);
Field field = new Field(TSDataType.TEXT);
@@ -351,10 +349,13 @@ public class PlanExecutor implements IPlanExecutor {
return listDataSet;
}
+ protected Set<String> getDevices(String path) throws MetadataException {
+ return MManager.getInstance().getDevices(path);
+ }
+
private QueryDataSet processShowChildPaths(ShowChildPathsPlan showChildPathsPlan)
throws MetadataException {
- Set<String> childPathsList = MManager.getInstance()
- .getChildNodePathInNextLevel(showChildPathsPlan.getPath().toString());
+ Set<String> childPathsList = getPathNextChildren(showChildPathsPlan.getPath().toString());
ListDataSet listDataSet = new ListDataSet(
Collections.singletonList(new Path(COLUMN_CHILD_PATHS)),
Collections.singletonList(TSDataType.TEXT));
@@ -368,6 +369,10 @@ public class PlanExecutor implements IPlanExecutor {
return listDataSet;
}
+ protected Set<String> getPathNextChildren(String path) throws MetadataException {
+ return MManager.getInstance().getChildNodePathInNextLevel(path);
+ }
+
private QueryDataSet processShowStorageGroup() {
ListDataSet listDataSet = new ListDataSet(
Collections.singletonList(new Path(COLUMN_STORAGE_GROUP)),
@@ -393,8 +398,7 @@ public class PlanExecutor implements IPlanExecutor {
new Path(COLUMN_TIMESERIES_COMPRESSION)),
Arrays.asList(TSDataType.TEXT, TSDataType.TEXT, TSDataType.TEXT, TSDataType.TEXT,
TSDataType.TEXT));
- List<String[]> timeseriesList = MManager.getInstance()
- .getAllTimeseriesSchema(timeSeriesPlan.getPath().toString());
+ List<String[]> timeseriesList = getTimeseriesSchemas(timeSeriesPlan.getPath().toString());
for (String[] list : timeseriesList) {
RowRecord record = new RowRecord(0);
for (String s : list) {
@@ -407,6 +411,10 @@ public class PlanExecutor implements IPlanExecutor {
return listDataSet;
}
+ protected List<String[]> getTimeseriesSchemas(String path) throws MetadataException {
+ return MManager.getInstance().getAllTimeseriesSchema(path);
+ }
+
private QueryDataSet processShowTTLQuery(ShowTTLPlan showTTLPlan) {
ListDataSet listDataSet = new ListDataSet(
Arrays.asList(new Path(COLUMN_STORAGE_GROUP), new Path(COLUMN_TTL))
@@ -506,18 +514,12 @@ public class PlanExecutor implements IPlanExecutor {
try {
Set<String> existingPaths = new HashSet<>();
for (Path p : deletePlan.getPaths()) {
- existingPaths.addAll(mManager.getAllTimeseriesName(p.getFullPath()));
+ existingPaths.addAll(getPaths(p.getFullPath()));
}
if (existingPaths.isEmpty()) {
throw new QueryProcessException(
"TimeSeries does not exist and its data cannot be deleted");
}
- for (String onePath : existingPaths) {
- if (!mManager.isPathExist(onePath)) {
- throw new QueryProcessException(String
- .format("TimeSeries %s does not exist and its data cannot be deleted", onePath));
- }
- }
for (String path : existingPaths) {
delete(new Path(path), deletePlan.getDeleteTime());
}
@@ -614,7 +616,7 @@ public class PlanExecutor implements IPlanExecutor {
.createTimeseries(fullPath, schema.getType(), schema.getEncodingType(),
schema.getCompressor(), Collections.emptyMap());
if (result) {
- storageEngine
+ StorageEngine.getInstance()
.addTimeSeries(new Path(fullPath), schema.getType(), schema.getEncodingType(),
schema.getCompressor(), Collections.emptyMap());
}
@@ -687,7 +689,7 @@ public class PlanExecutor implements IPlanExecutor {
String.format("Time series %s does not exist.", path.getFullPath()));
}
mManager.getStorageGroupName(path.getFullPath());
- storageEngine.delete(deviceId, measurementId, timestamp);
+ StorageEngine.getInstance().delete(deviceId, measurementId, timestamp);
} catch (MetadataException | StorageEngineException e) {
throw new QueryProcessException(e);
}
@@ -707,9 +709,7 @@ public class PlanExecutor implements IPlanExecutor {
String measurement = measurementList[i];
if (!node.hasChild(measurement)) {
if (!IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()) {
- throw new QueryProcessException(
- String.format("Current deviceId[%s] does not contain measurement:%s",
- deviceId, measurement));
+ throw new PathNotExistException(deviceId + PATH_SEPARATOR + measurement);
}
TSDataType dataType = TypeInferenceUtils.getPredictedDataType(strValues[i]);
Path path = new Path(deviceId, measurement);
@@ -719,7 +719,7 @@ public class PlanExecutor implements IPlanExecutor {
TSFileDescriptor.getInstance().getConfig().getCompressor(),
Collections.emptyMap());
if (result) {
- storageEngine.addTimeSeries(path, dataType, getDefaultEncoding(dataType));
+ StorageEngine.getInstance().addTimeSeries(path, dataType, getDefaultEncoding(dataType));
}
}
MNode measurementNode = node.getChild(measurement);
@@ -731,8 +731,8 @@ public class PlanExecutor implements IPlanExecutor {
dataTypes[i] = measurementNode.getSchema().getType();
}
insertPlan.setDataTypes(dataTypes);
- storageEngine.insert(insertPlan);
- } catch (PathException | StorageEngineException | MetadataException e) {
+ StorageEngine.getInstance().insert(insertPlan);
+ } catch (StorageEngineException | MetadataException e) {
throw new QueryProcessException(e);
}
}
@@ -785,7 +785,7 @@ public class PlanExecutor implements IPlanExecutor {
TSFileDescriptor.getInstance().getConfig().getCompressor(),
Collections.emptyMap());
if (result) {
- storageEngine
+ StorageEngine.getInstance()
.addTimeSeries(path, dataType, getDefaultEncoding(dataType));
}
}
@@ -803,8 +803,8 @@ public class PlanExecutor implements IPlanExecutor {
measurementNode.getSchema().getType()));
}
}
- return storageEngine.insertBatch(batchInsertPlan);
- } catch (PathException | StorageEngineException | MetadataException e) {
+ return StorageEngine.getInstance().insertBatch(batchInsertPlan);
+ } catch (StorageEngineException | MetadataException e) {
throw new QueryProcessException(e);
}
}
@@ -892,7 +892,7 @@ public class PlanExecutor implements IPlanExecutor {
boolean result = mManager
.createTimeseries(path.getFullPath(), dataType, encoding, compressor, props);
if (result) {
- storageEngine.addTimeSeries(path, dataType, encoding, compressor, props);
+ StorageEngine.getInstance().addTimeSeries(path, dataType, encoding, compressor, props);
}
} catch (StorageEngineException | MetadataException e) {
throw new QueryProcessException(e);
@@ -910,7 +910,7 @@ public class PlanExecutor implements IPlanExecutor {
emptyStorageGroups.addAll(mManager.deleteTimeseries(path.toString()));
}
for (String deleteStorageGroup : emptyStorageGroups) {
- storageEngine.deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
+ StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
}
} catch (MetadataException e) {
throw new QueryProcessException(e);
@@ -918,7 +918,7 @@ public class PlanExecutor implements IPlanExecutor {
return true;
}
- private boolean setStorageGroup(SetStorageGroupPlan setStorageGroupPlan)
+ public boolean setStorageGroup(SetStorageGroupPlan setStorageGroupPlan)
throws QueryProcessException {
Path path = setStorageGroupPlan.getPath();
try {
@@ -934,7 +934,7 @@ public class PlanExecutor implements IPlanExecutor {
List<String> deletePathList = new ArrayList<>();
try {
for (Path storageGroupPath : deleteStorageGroupPlan.getPaths()) {
- storageEngine.deleteStorageGroup(storageGroupPath.getFullPath());
+ StorageEngine.getInstance().deleteStorageGroup(storageGroupPath.getFullPath());
deletePathList.add(storageGroupPath.getFullPath());
}
mManager.deleteStorageGroups(deletePathList);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/FilterOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/FilterOperator.java
index 67f5946..bc22acb 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/FilterOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/FilterOperator.java
@@ -44,16 +44,16 @@ import org.apache.iotdb.tsfile.utils.StringContainer;
public class FilterOperator extends Operator implements Comparable<FilterOperator> {
// it is the symbol of token. e.g. AND is & and OR is |
- protected String tokenSymbol;
+ String tokenSymbol;
- protected List<FilterOperator> childOperators;
+ private List<FilterOperator> childOperators;
// leaf filter operator means it doesn't have left and right child filterOperator. Leaf filter
// should set FunctionOperator.
- protected boolean isLeaf = false;
+ protected boolean isLeaf;
// isSingle being true means all recursive children of this filter belong to one seriesPath.
- protected boolean isSingle = false;
+ boolean isSingle = false;
// if isSingle = false, singlePath must be null
- protected Path singlePath = null;
+ Path singlePath = null;
public FilterOperator(int tokenType) {
super(tokenType);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index c86a3e0..f2d3fc0 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.qp.physical;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -26,6 +27,8 @@ import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -34,10 +37,23 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public abstract class PhysicalPlan {
+ private static final String SERIALIZATION_UNIMPLEMENTED = "serialization unimplemented";
+
private boolean isQuery;
private Operator.OperatorType operatorType;
private static final int NULL_VALUE_LEN = -1;
+ //for cluster mode, whether the plan may be splitted into several sub plans
+ protected boolean canbeSplit = true;
+
+ /**
+ * whether the plan can be split into more than one Plans.
+ * Only used in the cluster mode.
+ */
+ public boolean canbeSplit() {
+ return canbeSplit;
+ }
+
protected PhysicalPlan(boolean isQuery) {
this.isQuery = isQuery;
}
@@ -73,12 +89,16 @@ public abstract class PhysicalPlan {
isQuery = query;
}
+ public void serializeTo(DataOutputStream stream) throws IOException {
+ throw new UnsupportedOperationException(SERIALIZATION_UNIMPLEMENTED);
+ }
+
public void serializeTo(ByteBuffer buffer) {
- throw new UnsupportedOperationException("serialize of unimplemented");
+ throw new UnsupportedOperationException(SERIALIZATION_UNIMPLEMENTED);
}
public void deserializeFrom(ByteBuffer buffer) {
- throw new UnsupportedOperationException("serialize of unimplemented");
+ throw new UnsupportedOperationException(SERIALIZATION_UNIMPLEMENTED);
}
protected void putString(ByteBuffer buffer, String value) {
@@ -89,6 +109,14 @@ public abstract class PhysicalPlan {
}
}
+ protected void putString(DataOutputStream stream, String value) throws IOException {
+ if (value == null) {
+ stream.writeInt(NULL_VALUE_LEN);
+ } else {
+ ReadWriteIOUtils.write(value, stream);
+ }
+ }
+
protected String readString(ByteBuffer buffer) {
int valueLen = buffer.getInt();
if (valueLen == NULL_VALUE_LEN) {
@@ -110,6 +138,7 @@ public abstract class PhysicalPlan {
}
PhysicalPlanType type = PhysicalPlanType.values()[typeNum];
PhysicalPlan plan;
+ // TODO-Cluster: support more plans
switch (type) {
case INSERT:
plan = new InsertPlan();
@@ -123,6 +152,14 @@ public abstract class PhysicalPlan {
plan = new BatchInsertPlan();
plan.deserializeFrom(buffer);
break;
+ case SET_STORAGE_GROUP:
+ plan = new SetStorageGroupPlan();
+ plan.deserializeFrom(buffer);
+ break;
+ case CREATE_TIMESERIES:
+ plan = new CreateTimeSeriesPlan();
+ plan.deserializeFrom(buffer);
+ break;
default:
throw new IOException("unrecognized log type " + type);
}
@@ -131,7 +168,7 @@ public abstract class PhysicalPlan {
}
public enum PhysicalPlanType {
- INSERT, DELETE, BATCHINSERT
+ INSERT, DELETE, BATCHINSERT, SET_STORAGE_GROUP, CREATE_TIMESERIES
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java
index 11382a4..07a4d97 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.qp.physical.crud;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
@@ -42,11 +44,13 @@ import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsLong;
public class BatchInsertPlan extends PhysicalPlan {
+ private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported.";
+
private String deviceId;
private String[] measurements;
private TSDataType[] dataTypes;
- private long[] times;
+ private long[] times; // times should be sorted. It is done in the session API.
private ByteBuffer timeBuffer;
private Object[] columns;
@@ -69,6 +73,11 @@ public class BatchInsertPlan extends PhysicalPlan {
this.deviceId = deviceId;
setMeasurements(measurements);
}
+ public BatchInsertPlan(String deviceId, String[] measurements) {
+ super(false, OperatorType.BATCHINSERT);
+ this.deviceId = deviceId;
+ setMeasurements(measurements);
+ }
public BatchInsertPlan(String deviceId, String[] measurements, List<Integer> dataTypes) {
super(false, OperatorType.BATCHINSERT);
@@ -115,6 +124,94 @@ public class BatchInsertPlan extends PhysicalPlan {
}
@Override
+ public void serializeTo(DataOutputStream stream) throws IOException {
+ int type = PhysicalPlanType.BATCHINSERT.ordinal();
+ stream.writeByte((byte) type);
+
+ putString(stream, deviceId);
+
+ stream.writeInt(measurements.length);
+ for (String m : measurements) {
+ putString(stream, m);
+ }
+
+ for (TSDataType dataType : dataTypes) {
+ stream.writeShort(dataType.serialize());
+ }
+
+ stream.writeInt(index.size());
+
+ if (timeBuffer == null) {
+ for(int loc : index){
+ stream.writeLong(times[loc]);
+ }
+ } else {
+ stream.write(timeBuffer.array());
+ timeBuffer = null;
+ }
+
+ if (valueBuffer == null) {
+ serializeValues(stream);
+ } else {
+ stream.write(valueBuffer.array());
+ valueBuffer = null;
+ }
+ }
+
+ private void serializeValues(DataOutputStream stream) throws IOException {
+ for (int i = 0; i < measurements.length; i++) {
+ serializeColumn(dataTypes[i], columns[i], stream, index);
+ }
+ }
+
+ private void serializeColumn(TSDataType dataType, Object column, DataOutputStream stream,
+ Set<Integer> index)
+ throws IOException {
+ switch (dataType) {
+ case INT32:
+ int[] intValues = (int[]) column;
+ for(int loc : index){
+ stream.writeInt(intValues[loc]);
+ }
+ break;
+ case INT64:
+ long[] longValues = (long[]) column;
+ for(int loc : index){
+ stream.writeLong(longValues[loc]);
+ }
+ break;
+ case FLOAT:
+ float[] floatValues = (float[]) column;
+ for(int loc : index){
+ stream.writeFloat(floatValues[loc]);
+ }
+ break;
+ case DOUBLE:
+ double[] doubleValues = (double[]) column;
+ for(int loc : index){
+ stream.writeDouble(doubleValues[loc]);
+ }
+ break;
+ case BOOLEAN:
+ boolean[] boolValues = (boolean[]) column;
+ for(int loc : index){
+ stream.write(BytesUtils.boolToByte(boolValues[loc]));
+ }
+ break;
+ case TEXT:
+ Binary[] binaryValues = (Binary[]) column;
+ for(int loc : index){
+ stream.writeInt(binaryValues[loc].getLength());
+ stream.write(binaryValues[loc].getValues());
+ }
+ break;
+ default:
+ throw new UnSupportedDataTypeException(
+ String.format(DATATYPE_UNSUPPORTED, dataType));
+ }
+ }
+
+ @Override
public void serializeTo(ByteBuffer buffer) {
int type = PhysicalPlanType.BATCHINSERT.ordinal();
buffer.put((byte) type);
@@ -142,57 +239,65 @@ public class BatchInsertPlan extends PhysicalPlan {
}
if (valueBuffer == null) {
- for (int i = 0; i < measurements.length; i++) {
- TSDataType dataType = dataTypes[i];
- switch (dataType) {
- case INT32:
- int[] intValues = (int[]) columns[i];
- for (int j = start; j < end; j++) {
- buffer.putInt(intValues[j]);
- }
- break;
- case INT64:
- long[] longValues = (long[]) columns[i];
- for (int j = start; j < end; j++) {
- buffer.putLong(longValues[j]);
- }
- break;
- case FLOAT:
- float[] floatValues = (float[]) columns[i];
- for (int j = start; j < end; j++) {
- buffer.putFloat(floatValues[j]);
- }
- break;
- case DOUBLE:
- double[] doubleValues = (double[]) columns[i];
- for (int j = start; j < end; j++) {
- buffer.putDouble(doubleValues[j]);
- }
- break;
- case BOOLEAN:
- boolean[] boolValues = (boolean[]) columns[i];
- for (int j = start; j < end; j++) {
- buffer.putInt(BytesUtils.boolToByte(boolValues[j]));
- }
- break;
- case TEXT:
- Binary[] binaryValues = (Binary[]) columns[i];
- for (int j = start; j < end; j++) {
- buffer.putInt(binaryValues[j].getLength());
- buffer.put(binaryValues[j].getValues());
- }
- break;
- default:
- throw new UnSupportedDataTypeException(
- String.format("Data type %s is not supported.", dataType));
- }
- }
+ serializeValues(buffer);
} else {
buffer.put(valueBuffer.array());
valueBuffer = null;
}
}
+ private void serializeValues(ByteBuffer buffer) {
+ for (int i = 0; i < measurements.length; i++) {
+ serializeColumn(dataTypes[i], columns[i], buffer, start, end);
+ }
+ }
+
+ private void serializeColumn(TSDataType dataType, Object column, ByteBuffer buffer,
+ int start, int end) {
+ switch (dataType) {
+ case INT32:
+ int[] intValues = (int[]) column;
+ for (int j = start; j < end; j++) {
+ buffer.putInt(intValues[j]);
+ }
+ break;
+ case INT64:
+ long[] longValues = (long[]) column;
+ for (int j = start; j < end; j++) {
+ buffer.putLong(longValues[j]);
+ }
+ break;
+ case FLOAT:
+ float[] floatValues = (float[]) column;
+ for (int j = start; j < end; j++) {
+ buffer.putFloat(floatValues[j]);
+ }
+ break;
+ case DOUBLE:
+ double[] doubleValues = (double[]) column;
+ for (int j = start; j < end; j++) {
+ buffer.putDouble(doubleValues[j]);
+ }
+ break;
+ case BOOLEAN:
+ boolean[] boolValues = (boolean[]) column;
+ for (int j = start; j < end; j++) {
+ buffer.putInt(BytesUtils.boolToByte(boolValues[j]));
+ }
+ break;
+ case TEXT:
+ Binary[] binaryValues = (Binary[]) column;
+ for (int j = start; j < end; j++) {
+ buffer.putInt(binaryValues[j].getLength());
+ buffer.put(binaryValues[j].getValues());
+ }
+ break;
+ default:
+ throw new UnSupportedDataTypeException(
+ String.format(DATATYPE_UNSUPPORTED, dataType));
+ }
+ }
+
public void setTimeBuffer(ByteBuffer timeBuffer) {
this.timeBuffer = timeBuffer;
this.timeBuffer.position(0);
@@ -259,6 +364,10 @@ public class BatchInsertPlan extends PhysicalPlan {
}
}
+ public void setDataTypes(TSDataType[] dataTypes) {
+ this.dataTypes = dataTypes;
+ }
+
public Object[] getColumns() {
return columns;
}
@@ -288,13 +397,13 @@ public class BatchInsertPlan extends PhysicalPlan {
if (maxTime != null) {
return maxTime;
}
- long maxTime = Long.MIN_VALUE;
+ long tmpMaxTime = Long.MIN_VALUE;
for (Long time : times) {
- if (time > maxTime) {
- maxTime = time;
+ if (time > tmpMaxTime) {
+ tmpMaxTime = time;
}
}
- return maxTime;
+ return tmpMaxTime;
}
public TimeValuePair composeLastTimeValuePair(int measurementIndex) {
@@ -329,7 +438,7 @@ public class BatchInsertPlan extends PhysicalPlan {
break;
default:
throw new UnSupportedDataTypeException(
- String.format("Data type %s is not supported.", dataTypes[measurementIndex]));
+ String.format(DATATYPE_UNSUPPORTED, dataTypes[measurementIndex]));
}
return new TimeValuePair(times[end - 1], value);
}
@@ -349,4 +458,5 @@ public class BatchInsertPlan extends PhysicalPlan {
public void setRowCount(int size) {
this.rowCount = size;
}
+
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/DeletePlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/DeletePlan.java
index a46aaea..d2f1236 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/DeletePlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/DeletePlan.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.qp.physical.crud;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
@@ -102,6 +104,14 @@ public class DeletePlan extends PhysicalPlan {
}
@Override
+ public void serializeTo(DataOutputStream stream) throws IOException {
+ int type = PhysicalPlanType.DELETE.ordinal();
+ stream.writeByte((byte) type);
+ stream.writeLong(deleteTime);
+ putString(stream, paths.get(0).getFullPath());
+ }
+
+ @Override
public void serializeTo(ByteBuffer buffer) {
int type = PhysicalPlanType.DELETE.ordinal();
buffer.put((byte) type);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
index 6544298..4d7fc53 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.qp.physical.crud;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -46,6 +48,7 @@ public class InsertPlan extends PhysicalPlan {
public InsertPlan() {
super(false, OperatorType.INSERT);
+ canbeSplit = false;
}
@TestOnly
@@ -55,6 +58,7 @@ public class InsertPlan extends PhysicalPlan {
this.deviceId = deviceId;
this.measurements = new String[] {measurement};
this.values = new String[] {insertValue};
+ canbeSplit = false;
}
public InsertPlan(TSRecord tsRecord) {
@@ -69,6 +73,7 @@ public class InsertPlan extends PhysicalPlan {
dataTypes[i] = tsRecord.dataPointList.get(i).getType();
values[i] = tsRecord.dataPointList.get(i).getValue().toString();
}
+ canbeSplit = false;
}
public InsertPlan(String deviceId, long insertTime, String[] measurementList,
@@ -78,6 +83,7 @@ public class InsertPlan extends PhysicalPlan {
this.deviceId = deviceId;
this.measurements = measurementList;
this.values = insertValues;
+ canbeSplit = false;
}
public long getTime() {
@@ -150,6 +156,25 @@ public class InsertPlan extends PhysicalPlan {
}
@Override
+ public void serializeTo(DataOutputStream stream) throws IOException {
+ int type = PhysicalPlanType.INSERT.ordinal();
+ stream.writeByte((byte) type);
+ stream.writeLong(time);
+
+ putString(stream, deviceId);
+
+ stream.writeInt(measurements.length);
+ for (String m : measurements) {
+ putString(stream, m);
+ }
+
+ stream.writeInt(values.length);
+ for (String m : values) {
+ putString(stream, m);
+ }
+ }
+
+ @Override
public void serializeTo(ByteBuffer buffer) {
int type = PhysicalPlanType.INSERT.ordinal();
buffer.put((byte) type);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CountPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CountPlan.java
index 044d838..b1851ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CountPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CountPlan.java
@@ -22,6 +22,9 @@ import org.apache.iotdb.tsfile.read.common.Path;
/**
* CountPlan is used to count time-series and count nodes.
+ * COUNT_TIMESERIES if using "COUNT TIMESERIES <Path>" and only this command supports wildcard.
+ * COUNT_NODE_TIMESERIES if using "COUNT TIMESERIES <Path> GROUP BY LEVEL=<INTEGER>"
+ * COUNT_NODE if using "COUNT NODES <Path> LEVEL=<INTEGER>"
*/
public class CountPlan extends ShowPlan {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java
index a6ce3cd..7e35ff9 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java
@@ -18,10 +18,13 @@
*/
package org.apache.iotdb.db.qp.physical.sys;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-
+import java.util.Objects;
import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -36,8 +39,13 @@ public class CreateTimeSeriesPlan extends PhysicalPlan {
private TSEncoding encoding;
private CompressionType compressor;
private Map<String, String> props;
-
- public CreateTimeSeriesPlan(Path path, TSDataType dataType, TSEncoding encoding,
+
+ public CreateTimeSeriesPlan() {
+ super(false, Operator.OperatorType.CREATE_TIMESERIES);
+ canbeSplit = false;
+ }
+
+ public CreateTimeSeriesPlan(Path path, TSDataType dataType, TSEncoding encoding,
CompressionType compressor, Map<String, String> props) {
super(false, Operator.OperatorType.CREATE_TIMESERIES);
this.path = path;
@@ -45,6 +53,7 @@ public class CreateTimeSeriesPlan extends PhysicalPlan {
this.encoding = encoding;
this.compressor = compressor;
this.props = props;
+ canbeSplit = false;
}
public Path getPath() {
@@ -89,14 +98,8 @@ public class CreateTimeSeriesPlan extends PhysicalPlan {
@Override
public String toString() {
- String ret = String.format("seriesPath: %s%nresultDataType: %s%nencoding: %s%nnamespace type:"
- + " ADD_PATH%nargs: ", path, dataType, encoding);
- StringBuilder stringBuilder = new StringBuilder(ret.length()+50);
- stringBuilder.append(ret);
- for (Map.Entry<String, String> prop : props.entrySet()) {
- stringBuilder.append(prop.getKey()).append("=").append(prop.getValue()).append(",");
- }
- return stringBuilder.toString();
+ return String.format("seriesPath: %s, resultDataType: %s, encoding: %s, compression: %s", path,
+ dataType, encoding, compressor);
}
@Override
@@ -104,4 +107,45 @@ public class CreateTimeSeriesPlan extends PhysicalPlan {
return Collections.singletonList(path);
}
+ @Override
+ public void serializeTo(DataOutputStream stream) throws IOException {
+ stream.writeByte((byte) PhysicalPlanType.CREATE_TIMESERIES.ordinal());
+ byte[] pathBytes = path.getFullPath().getBytes();
+ stream.writeInt(pathBytes.length);
+ stream.write(pathBytes);
+ stream.write(dataType.ordinal());
+ stream.write(encoding.ordinal());
+ stream.write(compressor.ordinal());
+ }
+
+ @Override
+ public void deserializeFrom(ByteBuffer buffer) {
+ int length = buffer.getInt();
+ byte[] pathBytes = new byte[length];
+ buffer.get(pathBytes);
+ path = new Path(new String(pathBytes));
+ dataType = TSDataType.values()[buffer.get()];
+ encoding = TSEncoding.values()[buffer.get()];
+ compressor = CompressionType.values()[buffer.get()];
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ CreateTimeSeriesPlan that = (CreateTimeSeriesPlan) o;
+ return Objects.equals(path, that.path) &&
+ dataType == that.dataType &&
+ encoding == that.encoding &&
+ compressor == that.compressor;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path, dataType, encoding, compressor);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/SetStorageGroupPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/SetStorageGroupPlan.java
index 72e9ebc..28d19cd 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/SetStorageGroupPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/SetStorageGroupPlan.java
@@ -18,16 +18,23 @@
*/
package org.apache.iotdb.db.qp.physical.sys;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-
+import java.util.Objects;
import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.tsfile.read.common.Path;
public class SetStorageGroupPlan extends PhysicalPlan {
private Path path;
-
+
+ public SetStorageGroupPlan() {
+ super(false, Operator.OperatorType.SET_STORAGE_GROUP);
+ }
+
public SetStorageGroupPlan(Path path) {
super(false, Operator.OperatorType.SET_STORAGE_GROUP);
this.path = path;
@@ -50,4 +57,41 @@ public class SetStorageGroupPlan extends PhysicalPlan {
return ret;
}
+ @Override
+ public void serializeTo(DataOutputStream stream) throws IOException {
+ stream.write((byte) PhysicalPlanType.SET_STORAGE_GROUP.ordinal());
+ byte[] fullPathBytes = path.getFullPath().getBytes();
+ stream.writeInt(fullPathBytes.length);
+ stream.write(fullPathBytes);
+ }
+
+ @Override
+ public void deserializeFrom(ByteBuffer buffer) {
+ int length = buffer.getInt();
+ byte[] fullPathBytes = new byte[length];
+ buffer.get(fullPathBytes);
+ path = new Path(new String(fullPathBytes));
+ }
+
+ @Override
+ public String toString() {
+ return "SetStorageGroup{" + path + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SetStorageGroupPlan that = (SetStorageGroupPlan) o;
+ return Objects.equals(path, that.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowChildPathsPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowChildPathsPlan.java
index 9f6c199..8490c6b 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowChildPathsPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowChildPathsPlan.java
@@ -22,14 +22,17 @@ package org.apache.iotdb.db.qp.physical.sys;
import org.apache.iotdb.tsfile.read.common.Path;
public class ShowChildPathsPlan extends ShowPlan {
- private Path path;
- public ShowChildPathsPlan(ShowContentType showContentType, Path path) {
+ // the path could be a prefix path with wildcard
+ private Path prefixPath;
+
+ public ShowChildPathsPlan(ShowContentType showContentType, Path prefixPath) {
super(showContentType);
- this.path = path;
+ this.prefixPath = prefixPath;
+ canbeSplit = false;
}
public Path getPath() {
- return this.path;
+ return this.prefixPath;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index e6abde7..1f2413e 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -83,7 +83,7 @@ import org.apache.iotdb.db.qp.physical.sys.ShowPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowPlan.ShowContentType;
import org.apache.iotdb.db.qp.physical.sys.ShowTTLPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
-import org.apache.iotdb.db.service.TSServiceImpl;
+import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.IExpression;
@@ -211,6 +211,9 @@ public class PhysicalGenerator {
}
}
+ protected TSDataType getSeriesType(String path) throws MetadataException {
+ return SchemaUtils.getSeriesType(path);
+ }
private PhysicalPlan transformQuery(QueryOperator queryOperator)
throws QueryProcessException {
@@ -290,8 +293,7 @@ public class PhysicalGenerator {
Path fullPath = Path.addPrefixPath(suffixPath, device);
try {
// remove stars in SELECT to get actual paths
- List<String> actualPaths = MManager.getInstance()
- .getAllTimeseriesName(fullPath.getFullPath());
+ List<String> actualPaths = getMatchedTimeseries(fullPath.getFullPath());
// for actual non exist path
if (actualPaths.isEmpty() && originAggregations.isEmpty()) {
@@ -313,7 +315,7 @@ public class PhysicalGenerator {
pathForDataType = path.getFullPath();
measurementChecked = path.getMeasurement();
}
- TSDataType dataType = TSServiceImpl.getSeriesType(pathForDataType);
+ TSDataType dataType = getSeriesType(pathForDataType);
if (dataTypeConsistencyChecker.containsKey(measurementChecked)) {
if (!dataType.equals(dataTypeConsistencyChecker.get(measurementChecked))) {
throw new QueryProcessException(
@@ -429,7 +431,7 @@ public class PhysicalGenerator {
Set<String> deviceSet = new LinkedHashSet<>();
try {
for (Path path : paths) {
- Set<String> tempDS = MManager.getInstance().getDevices(path.getFullPath());
+ Set<String> tempDS = getMatchedDevices(path.getFullPath());
deviceSet.addAll(tempDS);
}
retDevices = new ArrayList<>(deviceSet);
@@ -462,7 +464,7 @@ public class PhysicalGenerator {
List<Path> paths = queryPlan.getPaths();
List<TSDataType> dataTypes = new ArrayList<>(paths.size());
for (Path path : paths) {
- TSDataType seriesType = MManager.getInstance().getSeriesType(path.toString());
+ TSDataType seriesType = getSeriesType(path.toString());
dataTypes.add(seriesType);
queryPlan.addTypeMapping(path, seriesType);
}
@@ -530,5 +532,13 @@ public class PhysicalGenerator {
}
}
}
+
+ protected List<String> getMatchedTimeseries(String path) throws MetadataException {
+ return MManager.getInstance().getAllTimeseriesName(path);
+ }
+
+ protected Set<String> getMatchedDevices(String path) throws MetadataException {
+ return MManager.getInstance().getDevices(path);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 0217c0f..f71c60c 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -18,19 +18,26 @@
*/
package org.apache.iotdb.db.qp.strategy.optimizer;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.exception.runtime.SQLParserException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.logical.Operator;
-import org.apache.iotdb.db.qp.logical.crud.*;
+import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator;
+import org.apache.iotdb.db.qp.logical.crud.FilterOperator;
+import org.apache.iotdb.db.qp.logical.crud.FromOperator;
+import org.apache.iotdb.db.qp.logical.crud.FunctionOperator;
+import org.apache.iotdb.db.qp.logical.crud.QueryOperator;
+import org.apache.iotdb.db.qp.logical.crud.SFWOperator;
+import org.apache.iotdb.db.qp.logical.crud.SelectOperator;
import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-
/**
* concat paths in select and from clause.
*/
@@ -174,7 +181,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
* @param seriesLimit is ensured to be positive integer
* @param seriesOffset is ensured to be non-negative integer
*/
- public void slimitTrim(SelectOperator select, int seriesLimit, int seriesOffset)
+ private void slimitTrim(SelectOperator select, int seriesLimit, int seriesOffset)
throws LogicalOptimizeException {
List<Path> suffixList = select.getSuffixPaths();
List<String> aggregations = select.getAggregations();
@@ -268,7 +275,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
LinkedHashMap<String, Integer> pathMap = new LinkedHashMap<>();
try {
for (Path path : paths) {
- List<String> all = MManager.getInstance().getAllTimeseriesName(path.getFullPath());
+ List<String> all = removeWildcard(path.getFullPath());
for (String subPath : all) {
if (!pathMap.containsKey(subPath)) {
pathMap.put(subPath, 1);
@@ -290,7 +297,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
List<String> newAggregations = new ArrayList<>();
for (int i = 0; i < paths.size(); i++) {
try {
- List<String> actualPaths = MManager.getInstance().getAllTimeseriesName(paths.get(i).getFullPath());
+ List<String> actualPaths = removeWildcard(paths.get(i).getFullPath());
for (String actualPath : actualPaths) {
retPaths.add(new Path(actualPath));
if (afterConcatAggregations != null && !afterConcatAggregations.isEmpty()) {
@@ -304,4 +311,8 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
selectOperator.setSuffixPathList(retPaths);
selectOperator.setAggregations(newAggregations);
}
+
+ protected List<String> removeWildcard(String path) throws MetadataException {
+ return MManager.getInstance().getAllTimeseriesName(path);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/aggregation/AggregateResult.java b/server/src/main/java/org/apache/iotdb/db/query/aggregation/AggregateResult.java
index 5483bd3..ee083ee 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/aggregation/AggregateResult.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/aggregation/AggregateResult.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.query.aggregation;
+import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import org.apache.iotdb.db.exception.query.QueryProcessException;
@@ -29,8 +30,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.utils.Binary;
-
-import java.io.IOException;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public abstract class AggregateResult {
@@ -291,4 +290,8 @@ public abstract class AggregateResult {
return hasResult;
}
+ @Override
+ public String toString() {
+ return String.valueOf(getResult());
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index 03cbecc..a6d4653 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -105,7 +105,7 @@ public class QueryResourceManager {
try {
deserializer.close();
} catch (IOException e) {
- throw new StorageEngineException(e.getMessage());
+ throw new StorageEngineException(e);
}
}
externalSortFileMap.remove(queryId);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java
index 32d489f..7974bfb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java
@@ -201,7 +201,7 @@ public class AlignByDeviceDataSet extends QueryDataSet {
default:
throw new IOException("unsupported DataSetType");
}
- } catch (QueryProcessException | QueryFilterOptimizationException | StorageEngineException | IOException e) {
+ } catch (QueryProcessException | QueryFilterOptimizationException | StorageEngineException e) {
throw new IOException(e);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithValueFilter.java
index cd6a0b5..7859176 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithValueFilter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithValueFilter.java
@@ -21,16 +21,16 @@ package org.apache.iotdb.db.query.dataset;
import java.io.IOException;
import java.util.List;
import org.apache.iotdb.db.query.reader.series.IReaderByTimestamp;
-import org.apache.iotdb.db.query.timegenerator.ServerTimeGenerator;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Field;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator;
public class RawQueryDataSetWithValueFilter extends QueryDataSet {
- private ServerTimeGenerator timeGenerator;
+ private TimeGenerator timeGenerator;
private List<IReaderByTimestamp> seriesReaderByTimestampList;
private boolean hasCachedRowRecord;
private RowRecord cachedRowRecord;
@@ -44,7 +44,7 @@ public class RawQueryDataSetWithValueFilter extends QueryDataSet {
* @param readers readers in List(IReaderByTimeStamp) structure
*/
public RawQueryDataSetWithValueFilter(List<Path> paths, List<TSDataType> dataTypes,
- ServerTimeGenerator timeGenerator, List<IReaderByTimestamp> readers) {
+ TimeGenerator timeGenerator, List<IReaderByTimestamp> readers) {
super(paths, dataTypes);
this.timeGenerator = timeGenerator;
this.seriesReaderByTimestampList = readers;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
index 398ef9e..44402cc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
@@ -19,9 +19,11 @@
package org.apache.iotdb.db.query.dataset.groupby;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.qp.physical.crud.GroupByPlan;
import org.apache.iotdb.db.query.aggregation.AggregateResult;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -34,10 +36,6 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet {
private List<IReaderByTimestamp> allDataReaderList;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
index 9fa96f5..6882e7b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
@@ -28,7 +28,6 @@ import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.physical.crud.AggregationPlan;
import org.apache.iotdb.db.query.aggregation.AggregateResult;
@@ -36,11 +35,13 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.dataset.SingleDataSet;
import org.apache.iotdb.db.query.factory.AggregateResultFactory;
-import org.apache.iotdb.db.query.reader.series.IReaderByTimestamp;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.series.IAggregateReader;
+import org.apache.iotdb.db.query.reader.series.IReaderByTimestamp;
import org.apache.iotdb.db.query.reader.series.SeriesAggregateReader;
import org.apache.iotdb.db.query.reader.series.SeriesReaderByTimestamp;
import org.apache.iotdb.db.query.timegenerator.ServerTimeGenerator;
+import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.BatchData;
@@ -50,20 +51,21 @@ import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.impl.GlobalTimeExpression;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator;
public class AggregationExecutor {
private List<Path> selectedSeries;
- private List<TSDataType> dataTypes;
+ protected List<TSDataType> dataTypes;
private List<String> aggregations;
- private IExpression expression;
+ protected IExpression expression;
/**
* aggregation batch calculation size.
**/
private int aggregateFetchSize;
- public AggregationExecutor(AggregationPlan aggregationPlan) {
+ AggregationExecutor(AggregationPlan aggregationPlan) {
this.selectedSeries = aggregationPlan.getDeduplicatedPaths();
this.dataTypes = aggregationPlan.getDeduplicatedDataTypes();
this.aggregations = aggregationPlan.getDeduplicatedAggregations();
@@ -76,7 +78,7 @@ public class AggregationExecutor {
*
* @param context query context
*/
- public QueryDataSet executeWithoutValueFilter(QueryContext context)
+ QueryDataSet executeWithoutValueFilter(QueryContext context)
throws StorageEngineException, IOException, QueryProcessException {
Filter timeFilter = null;
@@ -112,87 +114,112 @@ public class AggregationExecutor {
Filter timeFilter, QueryContext context)
throws IOException, QueryProcessException, StorageEngineException {
List<AggregateResult> aggregateResultList = new ArrayList<>();
- List<Boolean> isCalculatedList = new ArrayList<>();
+
Path seriesPath = pathToAggrIndexes.getKey();
TSDataType tsDataType = dataTypes.get(pathToAggrIndexes.getValue().get(0));
+ for (int i : pathToAggrIndexes.getValue()) {
+ // construct AggregateResult
+ AggregateResult aggregateResult = AggregateResultFactory
+ .getAggrResultByName(aggregations.get(i), tsDataType);
+ aggregateResultList.add(aggregateResult);
+ }
+ aggregateOneSeries(seriesPath, context, timeFilter, tsDataType, aggregateResultList, null);
+ return aggregateResultList;
+ }
+
+ private static void aggregateOneSeries(Path seriesPath, QueryContext context, Filter timeFilter,
+ TSDataType tsDataType, List<AggregateResult> aggregateResultList, TsFileFilter fileFilter)
+ throws StorageEngineException, IOException, QueryProcessException {
+
// construct series reader without value filter
QueryDataSource queryDataSource = QueryResourceManager.getInstance()
.getQueryDataSource(seriesPath, context, timeFilter);
+ if (fileFilter != null) {
+ QueryUtils.filterQueryDataSource(queryDataSource, fileFilter);
+ }
// update filter by TTL
timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
- IAggregateReader seriesReader = new SeriesAggregateReader(pathToAggrIndexes.getKey(),
+ IAggregateReader seriesReader = new SeriesAggregateReader(seriesPath,
tsDataType, context, queryDataSource, timeFilter, null, null);
+ aggregateFromReader(seriesReader, aggregateResultList);
+ }
- for (int i : pathToAggrIndexes.getValue()) {
- // construct AggregateResult
- AggregateResult aggregateResult = AggregateResultFactory
- .getAggrResultByName(aggregations.get(i), tsDataType);
- aggregateResultList.add(aggregateResult);
- isCalculatedList.add(false);
- }
- int remainingToCalculate = pathToAggrIndexes.getValue().size();
+ private static void aggregateFromReader(IAggregateReader seriesReader,
+ List<AggregateResult> aggregateResultList) throws QueryProcessException, IOException {
+ int remainingToCalculate = aggregateResultList.size();
+ boolean[] isCalculatedArray = new boolean[aggregateResultList.size()];
while (seriesReader.hasNextChunk()) {
// cal by chunk statistics
if (seriesReader.canUseCurrentChunkStatistics()) {
Statistics chunkStatistics = seriesReader.currentChunkStatistics();
- for (int i = 0; i < aggregateResultList.size(); i++) {
- if (Boolean.FALSE.equals(isCalculatedList.get(i))) {
- AggregateResult aggregateResult = aggregateResultList.get(i);
- aggregateResult.updateResultFromStatistics(chunkStatistics);
- if (aggregateResult.isCalculatedAggregationResult()) {
- isCalculatedList.set(i, true);
- remainingToCalculate--;
- if (remainingToCalculate == 0) {
- return aggregateResultList;
- }
- }
- }
+ remainingToCalculate = aggregateStatistics(aggregateResultList, isCalculatedArray,
+ remainingToCalculate, chunkStatistics);
+ if (remainingToCalculate == 0) {
+ return;
}
seriesReader.skipCurrentChunk();
continue;
}
- while (seriesReader.hasNextPage()) {
- //cal by page statistics
- if (seriesReader.canUseCurrentPageStatistics()) {
- Statistics pageStatistic = seriesReader.currentPageStatistics();
- for (int i = 0; i < aggregateResultList.size(); i++) {
- if (Boolean.FALSE.equals(isCalculatedList.get(i))) {
- AggregateResult aggregateResult = aggregateResultList.get(i);
- aggregateResult.updateResultFromStatistics(pageStatistic);
- if (aggregateResult.isCalculatedAggregationResult()) {
- isCalculatedList.set(i, true);
- remainingToCalculate--;
- if (remainingToCalculate == 0) {
- return aggregateResultList;
- }
- }
- }
+ remainingToCalculate = aggregateOverlappedPages(seriesReader, aggregateResultList,
+ isCalculatedArray, remainingToCalculate);
+ }
+ }
+
+ /**
+ * Aggregate each result in the list with the statistics
+ * @param aggregateResultList
+ * @param isCalculatedArray
+ * @param remainingToCalculate
+ * @param statistics
+ * @return new remainingToCalculate
+ * @throws QueryProcessException
+ */
+ private static int aggregateStatistics(List<AggregateResult> aggregateResultList,
+ boolean[] isCalculatedArray, int remainingToCalculate, Statistics statistics)
+ throws QueryProcessException {
+ int newRemainingToCalculate = remainingToCalculate;
+ for (int i = 0; i < aggregateResultList.size(); i++) {
+ if (!isCalculatedArray[i]) {
+ AggregateResult aggregateResult = aggregateResultList.get(i);
+ aggregateResult.updateResultFromStatistics(statistics);
+ if (aggregateResult.isCalculatedAggregationResult()) {
+ isCalculatedArray[i] = true;
+ newRemainingToCalculate--;
+ if (newRemainingToCalculate == 0) {
+ return newRemainingToCalculate;
}
- seriesReader.skipCurrentPage();
- } else {
- // cal by page data
- BatchData batchData = seriesReader.nextPage();
- for (int i = 0; i < aggregateResultList.size(); i++) {
- if (Boolean.FALSE.equals(isCalculatedList.get(i))) {
- AggregateResult aggregateResult = aggregateResultList.get(i);
- aggregateResult.updateResultFromPageData(batchData);
- batchData.resetBatchData();
- if (aggregateResult.isCalculatedAggregationResult()) {
- isCalculatedList.set(i, true);
- remainingToCalculate--;
- if (remainingToCalculate == 0) {
- return aggregateResultList;
- }
- }
+ }
+ }
+ }
+ return newRemainingToCalculate;
+ }
+
+ private static int aggregateOverlappedPages(IAggregateReader seriesReader,
+ List<AggregateResult> aggregateResultList, boolean[] isCalculatedArray, int remainingToCalculate)
+ throws IOException {
+ // cal by page data
+ int newRemainingToCalculate = remainingToCalculate;
+ while (seriesReader.hasNextPage()) {
+ BatchData nextOverlappedPageData = seriesReader.nextPage();
+ for (int i = 0; i < aggregateResultList.size(); i++) {
+ if (!isCalculatedArray[i]) {
+ AggregateResult aggregateResult = aggregateResultList.get(i);
+ aggregateResult.updateResultFromPageData(nextOverlappedPageData);
+ nextOverlappedPageData.resetBatchData();
+ if (aggregateResult.isCalculatedAggregationResult()) {
+ isCalculatedArray[i] = true;
+ newRemainingToCalculate--;
+ if (newRemainingToCalculate == 0) {
+ return newRemainingToCalculate;
}
}
}
}
}
- return aggregateResultList;
+ return newRemainingToCalculate;
}
/**
@@ -200,16 +227,15 @@ public class AggregationExecutor {
*
* @param context query context.
*/
- public QueryDataSet executeWithValueFilter(QueryContext context)
- throws StorageEngineException, PathException, IOException {
+ QueryDataSet executeWithValueFilter(QueryContext context)
+ throws StorageEngineException, IOException {
- ServerTimeGenerator timestampGenerator = new ServerTimeGenerator(expression, context);
+ TimeGenerator timestampGenerator = getTimeGenerator(context);
List<IReaderByTimestamp> readersOfSelectedSeries = new ArrayList<>();
for (int i = 0; i < selectedSeries.size(); i++) {
Path path = selectedSeries.get(i);
- SeriesReaderByTimestamp seriesReaderByTimestamp = new SeriesReaderByTimestamp(path,
- dataTypes.get(i), context,
- QueryResourceManager.getInstance().getQueryDataSource(path, context, null), null);
+ IReaderByTimestamp seriesReaderByTimestamp = getReaderByTime(path,
+ dataTypes.get(i), context);
readersOfSelectedSeries.add(seriesReaderByTimestamp);
}
@@ -223,11 +249,22 @@ public class AggregationExecutor {
return constructDataSet(aggregateResults);
}
+ private TimeGenerator getTimeGenerator(QueryContext context) throws StorageEngineException {
+ return new ServerTimeGenerator(expression, context);
+ }
+
+ private IReaderByTimestamp getReaderByTime(Path path, TSDataType dataType,
+ QueryContext context) throws StorageEngineException {
+ return new SeriesReaderByTimestamp(path,
+ dataType, context,
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, null), null);
+ }
+
/**
* calculate aggregation result with value filter.
*/
private void aggregateWithValueFilter(List<AggregateResult> aggregateResults,
- ServerTimeGenerator timestampGenerator, List<IReaderByTimestamp> readersOfSelectedSeries)
+ TimeGenerator timestampGenerator, List<IReaderByTimestamp> readersOfSelectedSeries)
throws IOException {
while (timestampGenerator.hasNext()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java b/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java
index 7d9c47d..9807933 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java
@@ -65,7 +65,7 @@ public class QueryRouter implements IQueryRouter {
}
queryPlan.setExpression(optimizedExpression);
- RawDataQueryExecutor rawDataQueryExecutor = new RawDataQueryExecutor(queryPlan);
+ RawDataQueryExecutor rawDataQueryExecutor = getRawDataQueryExecutor(queryPlan);
if (!queryPlan.isAlignByTime()) {
return rawDataQueryExecutor.executeNonAlign(context);
@@ -79,6 +79,10 @@ public class QueryRouter implements IQueryRouter {
return rawDataQueryExecutor.executeWithoutValueFilter(context);
}
+ protected RawDataQueryExecutor getRawDataQueryExecutor(RawDataQueryPlan queryPlan) {
+ return new RawDataQueryExecutor(queryPlan);
+ }
+
@Override
public QueryDataSet aggregate(AggregationPlan aggregationPlan, QueryContext context)
throws QueryFilterOptimizationException, StorageEngineException, QueryProcessException,
@@ -94,7 +98,7 @@ public class QueryRouter implements IQueryRouter {
aggregationPlan.setExpression(optimizedExpression);
- AggregationExecutor engineExecutor = new AggregationExecutor(aggregationPlan);
+ AggregationExecutor engineExecutor = getAggregationExecutor(aggregationPlan);
if (optimizedExpression != null
&& optimizedExpression.getType() != ExpressionType.GLOBAL_TIME) {
@@ -104,6 +108,9 @@ public class QueryRouter implements IQueryRouter {
return engineExecutor.executeWithoutValueFilter(context);
}
+ protected AggregationExecutor getAggregationExecutor(AggregationPlan aggregationPlan) {
+ return new AggregationExecutor(aggregationPlan);
+ }
@Override
public QueryDataSet groupBy(GroupByPlan groupByPlan, QueryContext context)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java
index 8785f54..4f9ea11 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java
@@ -25,8 +25,8 @@ import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.qp.physical.crud.RawDataQueryPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
-import org.apache.iotdb.db.query.dataset.RawQueryDataSetWithValueFilter;
import org.apache.iotdb.db.query.dataset.NonAlignEngineDataSet;
+import org.apache.iotdb.db.query.dataset.RawQueryDataSetWithValueFilter;
import org.apache.iotdb.db.query.dataset.RawQueryDataSetWithoutValueFilter;
import org.apache.iotdb.db.query.reader.series.IReaderByTimestamp;
import org.apache.iotdb.db.query.reader.series.ManagedSeriesReader;
@@ -39,15 +39,16 @@ import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.impl.GlobalTimeExpression;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator;
/**
* IoTDB query executor.
*/
public class RawDataQueryExecutor {
- private List<Path> deduplicatedPaths;
- private List<TSDataType> deduplicatedDataTypes;
- private IExpression optimizedExpression;
+ protected List<Path> deduplicatedPaths;
+ protected List<TSDataType> deduplicatedDataTypes;
+ protected IExpression optimizedExpression;
public RawDataQueryExecutor(RawDataQueryPlan queryPlan) {
this.deduplicatedPaths = queryPlan.getDeduplicatedPaths();
@@ -77,7 +78,7 @@ public class RawDataQueryExecutor {
readersOfSelectedSeries);
}
- private List<ManagedSeriesReader> initManagedSeriesReader(QueryContext context)
+ protected List<ManagedSeriesReader> initManagedSeriesReader(QueryContext context)
throws StorageEngineException {
Filter timeFilter = null;
if (optimizedExpression != null) {
@@ -108,19 +109,30 @@ public class RawDataQueryExecutor {
*/
public QueryDataSet executeWithValueFilter(QueryContext context) throws StorageEngineException {
- ServerTimeGenerator timestampGenerator = new ServerTimeGenerator(
+ TimeGenerator timestampGenerator = getTimeGenerator(
optimizedExpression, context);
List<IReaderByTimestamp> readersOfSelectedSeries = new ArrayList<>();
for (int i = 0; i < deduplicatedPaths.size(); i++) {
Path path = deduplicatedPaths.get(i);
- SeriesReaderByTimestamp seriesReaderByTimestamp = new SeriesReaderByTimestamp(path,
- deduplicatedDataTypes.get(i), context,
- QueryResourceManager.getInstance().getQueryDataSource(path, context, null), null);
+ IReaderByTimestamp seriesReaderByTimestamp = getReaderByTimestamp(path,
+ deduplicatedDataTypes.get(i), context);
readersOfSelectedSeries.add(seriesReaderByTimestamp);
}
return new RawQueryDataSetWithValueFilter(deduplicatedPaths, deduplicatedDataTypes,
timestampGenerator, readersOfSelectedSeries);
}
+ protected IReaderByTimestamp getReaderByTimestamp(Path path, TSDataType dataType,
+ QueryContext context)
+ throws StorageEngineException {
+ return new SeriesReaderByTimestamp(path,
+ dataType, context,
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, null), null);
+ }
+
+ protected TimeGenerator getTimeGenerator(IExpression expression,
+ QueryContext context) throws StorageEngineException {
+ return new ServerTimeGenerator(expression, context);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJob.java b/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJob.java
index 4d91289..bf05fba 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJob.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJob.java
@@ -26,7 +26,7 @@ import org.apache.iotdb.tsfile.read.reader.IPointReader;
/**
* This class represents an external sort job. Every job will use a separated directory.
*/
-public class ExternalSortJob {
+class ExternalSortJob {
private List<ExternalSortJobPart> partList;
@@ -41,5 +41,4 @@ public class ExternalSortJob {
}
return readers;
}
-
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJobScheduler.java b/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJobScheduler.java
index 1db8cd3..c1e7071 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJobScheduler.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/externalsort/ExternalSortJobScheduler.java
@@ -16,29 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.query.externalsort;
+ package org.apache.iotdb.db.query.externalsort;
-import java.util.concurrent.atomic.AtomicLong;
+ public class ExternalSortJobScheduler {
-public class ExternalSortJobScheduler {
-
- private AtomicLong jobId = new AtomicLong(0);
+ private long queryId = 0;
private ExternalSortJobScheduler() {
}
- long genJobId() {
- return jobId.getAndIncrement();
+ synchronized long genJobId() {
+ queryId++;
+ return queryId;
}
private static class ExternalSortJobSchedulerHelper {
- private static final ExternalSortJobScheduler INSTANCE = new ExternalSortJobScheduler();
+ private static final ExternalSortJobScheduler INSTANCE = new ExternalSortJobScheduler();
}
public static ExternalSortJobScheduler getInstance() {
- return ExternalSortJobSchedulerHelper.INSTANCE;
+ return ExternalSortJobSchedulerHelper.INSTANCE;
}
-}
+ }
diff --git a/server/src/main/java/org/apache/iotdb/db/query/externalsort/SimpleExternalSortEngine.java b/server/src/main/java/org/apache/iotdb/db/query/externalsort/SimpleExternalSortEngine.java
index 5fba08b..00097a8 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/externalsort/SimpleExternalSortEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/externalsort/SimpleExternalSortEngine.java
@@ -103,10 +103,10 @@ public class SimpleExternalSortEngine implements ExternalSortJobEngine {
int toIndex = Math.min(i + minExternalSortSourceCount, ret.size());
List<ExternalSortJobPart> partGroup = ret.subList(i, toIndex);
i = toIndex;
- StringBuilder tmpFilePath = new StringBuilder(queryDir).append(jobId).append("_")
- .append(partId);
+ String tmpFilePath = queryDir + jobId + "_"
+ + partId;
MultiSourceExternalSortJobPart part = new MultiSourceExternalSortJobPart(queryId,
- tmpFilePath.toString(), partGroup);
+ tmpFilePath, partGroup);
tmpPartList.add(part);
partId++;
}
@@ -115,19 +115,19 @@ public class SimpleExternalSortEngine implements ExternalSortJobEngine {
return new ExternalSortJob(ret);
}
- public String getQueryDir() {
+ String getQueryDir() {
return queryDir;
}
- public void setQueryDir(String queryDir) {
+ void setQueryDir(String queryDir) {
this.queryDir = queryDir;
}
- public int getMinExternalSortSourceCount() {
+ int getMinExternalSortSourceCount() {
return minExternalSortSourceCount;
}
- public void setMinExternalSortSourceCount(int minExternalSortSourceCount) {
+ void setMinExternalSortSourceCount(int minExternalSortSourceCount) {
this.minExternalSortSourceCount = minExternalSortSourceCount;
}
@@ -171,10 +171,10 @@ public class SimpleExternalSortEngine implements ExternalSortJobEngine {
private static class SimpleExternalSortJobEngineHelper {
- private static SimpleExternalSortEngine INSTANCE = new SimpleExternalSortEngine();
+ private static final SimpleExternalSortEngine INSTANCE = new SimpleExternalSortEngine();
}
public static SimpleExternalSortEngine getInstance() {
return SimpleExternalSortJobEngineHelper.INSTANCE;
}
-}
\ No newline at end of file
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
index b15b72f..f9d43ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReaderByTimestamp.java
@@ -40,6 +40,10 @@ public class SeriesReaderByTimestamp implements IReaderByTimestamp {
dataSource, TimeFilter.gtEq(Long.MIN_VALUE), null, fileFilter);
}
+ public SeriesReaderByTimestamp(SeriesReader seriesReader) {
+ this.seriesReader = seriesReader;
+ }
+
@Override
public Object[] getValuesInTimestamps(long[] timestamps) throws IOException {
Object[] result = new Object[timestamps.length];
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java
index c5a7eba..4ebf7d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java
@@ -129,7 +129,7 @@ public class PriorityMergeReader implements IPointReader {
}
}
- class Element {
+ static class Element {
IPointReader reader;
TimeValuePair timeValuePair;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java
index 28fcd4e..5695e3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java
@@ -39,7 +39,11 @@ import org.apache.iotdb.tsfile.read.reader.IBatchReader;
*/
public class ServerTimeGenerator extends TimeGenerator {
- private QueryContext context;
+ protected QueryContext context;
+
+ public ServerTimeGenerator(QueryContext context) {
+ this.context = context;
+ }
/**
* Constructor of EngineTimeGenerator.
@@ -50,7 +54,7 @@ public class ServerTimeGenerator extends TimeGenerator {
try {
super.constructNode(expression);
} catch (IOException e) {
- throw new StorageEngineException(e.getMessage());
+ throw new StorageEngineException(e);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 0d2c4c5..072e2cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -96,7 +96,9 @@ public class IoTDB implements IoTDBMBean {
registerManager.register(UpgradeSevice.getINSTANCE());
registerManager.register(MergeManager.getINSTANCE());
registerManager.register(CacheHitRatioMonitor.getInstance());
- registerManager.register(MetricsService.getInstance());
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableMetricService()) {
+ registerManager.register(MetricsService.getInstance());
+ }
registerManager.register(StorageEngine.getInstance());
JMXService.registerMBean(getInstance(), mbeanName);
diff --git a/server/src/main/java/org/apache/iotdb/db/service/MetricsService.java b/server/src/main/java/org/apache/iotdb/db/service/MetricsService.java
index 6994886..90541fe 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/MetricsService.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/MetricsService.java
@@ -80,7 +80,7 @@ public class MetricsService implements MetricsServiceMBean, IService {
@Override
public synchronized void startService() throws StartupException {
- if (!IoTDBDescriptor.getInstance().getConfig().isEnableMetricsWebService()) {
+ if (!IoTDBDescriptor.getInstance().getConfig().isEnableMetricService()) {
return;
}
logger.info("{}: start {}...", IoTDBConstant.GLOBAL_DB_NAME, this.getID().getName());
diff --git a/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java b/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java
index 670ea32..0b0dea5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java
@@ -43,7 +43,8 @@ public enum ServiceType {
generateJmxName(IoTDBConstant.IOTDB_PACKAGE, "Cache Hit Ratio")),
FLUSH_SERVICE("Flush ServerService",
- generateJmxName("org.apache.iotdb.db.engine.pool", "Flush Manager"));
+ generateJmxName("org.apache.iotdb.db.engine.pool", "Flush Manager")),
+ CLUSTER_MONITOR_SERVICE("Cluster Monitor ServerService", "Cluster Monitor");
private String name;
private String jmxName;
diff --git a/server/src/main/java/org/apache/iotdb/db/service/StaticResps.java b/server/src/main/java/org/apache/iotdb/db/service/StaticResps.java
index c600551..441a73d 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/StaticResps.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/StaticResps.java
@@ -118,11 +118,21 @@ class StaticResps {
Collections.singletonList(COLUMN_PRIVILEGE),
Collections.singletonList(TSDataType.TEXT.toString()));
+ static final TSExecuteStatementResp LAST_RESP = getExecuteResp(
+ Arrays.asList(COLUMN_TIMESERIES, COLUMN_VALUE),
+ Arrays.asList(TSDataType.TEXT.toString(), TSDataType.TEXT.toString()), false
+ );
+
private static TSExecuteStatementResp getNoTimeExecuteResp(List<String> columns,
List<String> dataTypes) {
+ return getExecuteResp(columns, dataTypes, true);
+ }
+
+ private static TSExecuteStatementResp getExecuteResp(List<String> columns,
+ List<String> dataTypes, boolean ignoreTimeStamp) {
TSExecuteStatementResp resp =
TSServiceImpl.getTSExecuteStatementResp(TSServiceImpl.getStatus(TSStatusCode.SUCCESS_STATUS));
- resp.setIgnoreTimeStamp(true);
+ resp.setIgnoreTimeStamp(ignoreTimeStamp);
resp.setColumns(columns);
resp.setDataTypeList(dataTypes);
return resp;
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 3386c40..071181b 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -18,14 +18,6 @@
*/
package org.apache.iotdb.db.service;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PRIVILEGE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ROLE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_STORAGE_GROUP;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TTL;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_USER;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_VALUE;
-import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TIMESERIES;
-
import java.io.IOException;
import java.nio.ByteBuffer;
import java.sql.SQLException;
@@ -67,7 +59,12 @@ import org.apache.iotdb.db.qp.executor.IPlanExecutor;
import org.apache.iotdb.db.qp.executor.PlanExecutor;
import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
-import org.apache.iotdb.db.qp.physical.crud.*;
+import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
+import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.qp.physical.crud.LastQueryPlan;
+import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
import org.apache.iotdb.db.qp.physical.sys.AuthorPlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan;
@@ -81,6 +78,7 @@ import org.apache.iotdb.db.query.dataset.RawQueryDataSetWithoutValueFilter;
import org.apache.iotdb.db.tools.watermark.GroupedLSBWatermarkEncoder;
import org.apache.iotdb.db.tools.watermark.WatermarkEncoder;
import org.apache.iotdb.db.utils.QueryDataSetUtils;
+import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.ServerProperties;
import org.apache.iotdb.service.rpc.thrift.TSBatchInsertionReq;
@@ -137,8 +135,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
"meet error while parsing SQL to physical plan: {}";
public static Vector<SqlArgument> sqlArgumentsList = new Vector<>();
- private Planner processor;
- private IPlanExecutor executor;
+ protected Planner processor;
+ protected IPlanExecutor executor;
// Record the username for every rpc connection (session).
private Map<Long, String> sessionIdUsernameMap = new ConcurrentHashMap<>();
@@ -167,52 +165,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
executor = new PlanExecutor();
}
- public static TSDataType getSeriesType(String path) throws QueryProcessException {
- switch (path.toLowerCase()) {
- // authorization queries
- case COLUMN_ROLE:
- case COLUMN_USER:
- case COLUMN_PRIVILEGE:
- case COLUMN_STORAGE_GROUP:
- return TSDataType.TEXT;
- case COLUMN_TTL:
- return TSDataType.INT64;
- default:
- // do nothing
- }
-
- if (path.contains("(") && !path.startsWith("(") && path.endsWith(")")) {
- // aggregation
- int leftBracketIndex = path.indexOf('(');
- String aggrType = path.substring(0, leftBracketIndex);
- String innerPath = path.substring(leftBracketIndex + 1, path.length() - 1);
- switch (aggrType.toLowerCase()) {
- case SQLConstant.MIN_TIME:
- case SQLConstant.MAX_TIME:
- case SQLConstant.COUNT:
- return TSDataType.INT64;
- case SQLConstant.LAST_VALUE:
- case SQLConstant.FIRST_VALUE:
- case SQLConstant.MIN_VALUE:
- case SQLConstant.MAX_VALUE:
- return getSeriesType(innerPath);
- case SQLConstant.AVG:
- case SQLConstant.SUM:
- return TSDataType.DOUBLE;
- default:
- throw new QueryProcessException("aggregate does not support " + aggrType + " function.");
- }
- }
- TSDataType dataType;
- try {
- dataType = MManager.getInstance().getSeriesType(path);
- } catch (MetadataException e) {
- throw new QueryProcessException(e);
- }
-
- return dataType;
- }
-
@Override
public TSOpenSessionResp openSession(TSOpenSessionReq req) throws TException {
logger.info(
@@ -349,7 +301,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
/**
* release single operation resource
*/
- private void releaseQueryResource(long queryId) throws StorageEngineException {
+ protected void releaseQueryResource(long queryId) throws StorageEngineException {
// remove the corresponding Physical Plan
queryId2DataSet.remove(queryId);
QueryResourceManager.getInstance().endQuery(queryId);
@@ -376,6 +328,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
return new TSStatus(statusCodeAndMessage);
}
+ protected TSDataType getSeriesType(String path) throws QueryProcessException, MetadataException {
+ try {
+ return SchemaUtils.getSeriesType(path);
+ } catch (MetadataException e) {
+ throw new QueryProcessException(e);
+ }
+ }
+
@Override
public TSFetchMetadataResp fetchMetadata(TSFetchMetadataReq req) {
TSStatus status;
@@ -405,7 +365,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
status = getStatus(TSStatusCode.METADATA_ERROR, req.getType());
break;
}
- } catch (QueryProcessException | MetadataException | OutOfMemoryError e) {
+ } catch (MetadataException | OutOfMemoryError | QueryProcessException e) {
logger.error(
String.format("Failed to fetch timeseries %s's metadata", req.getColumnPath()), e);
status = getStatus(TSStatusCode.METADATA_ERROR, e.getMessage());
@@ -620,14 +580,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
long statementId, PhysicalPlan plan, int fetchSize, String username) {
long t1 = System.currentTimeMillis();
try {
- TSExecuteStatementResp resp; // column headers
- if (plan instanceof AuthorPlan) {
- resp = getAuthQueryColumnHeaders(plan);
- } else if (plan instanceof ShowPlan) {
- resp = getShowQueryColumnHeaders((ShowPlan) plan);
- } else {
- resp = getQueryColumnHeaders(plan, username);
- }
+ TSExecuteStatementResp resp = getQueryResp(plan, username); // column headers
+
if (plan instanceof QueryPlan && !((QueryPlan) plan).isAlignByTime()) {
if (plan.getOperatorType() == OperatorType.AGGREGATION) {
throw new QueryProcessException("Aggregation doesn't support disable align clause.");
@@ -654,12 +608,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (plan instanceof QueryPlan && !((QueryPlan) plan).isAlignByTime()) {
TSQueryNonAlignDataSet result = fillRpcNonAlignReturnData(fetchSize, newDataSet, username);
resp.setNonAlignQueryDataSet(result);
- resp.setQueryId(queryId);
} else {
TSQueryDataSet result = fillRpcReturnData(fetchSize, newDataSet, username);
resp.setQueryDataSet(result);
- resp.setQueryId(queryId);
}
+ resp.setQueryId(queryId);
return resp;
} catch (Exception e) {
logger.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e);
@@ -670,6 +623,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
}
+ private TSExecuteStatementResp getQueryResp(PhysicalPlan plan, String username)
+ throws QueryProcessException, AuthException, TException {
+ if (plan instanceof AuthorPlan) {
+ return getAuthQueryColumnHeaders(plan);
+ } else if (plan instanceof ShowPlan) {
+ return getShowQueryColumnHeaders((ShowPlan) plan);
+ } else {
+ return getQueryColumnHeaders(plan, username);
+ }
+ }
+
@Override
public TSExecuteStatementResp executeQueryStatement(TSExecuteStatementReq req) {
long startTime = System.currentTimeMillis();
@@ -784,7 +748,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (plan instanceof AlignByDevicePlan) {
getAlignByDeviceQueryHeaders((AlignByDevicePlan) plan, respColumns, columnsTypes);
} else if (plan instanceof LastQueryPlan) {
- getLastQueryHeaders(plan, respColumns, columnsTypes);
+ return StaticResps.LAST_RESP;
} else {
getWideQueryHeaders(plan, respColumns, columnsTypes);
}
@@ -824,7 +788,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
for (String column : respColumns) {
- columnTypes.add(getSeriesType(column).toString());
+ try {
+ columnTypes.add(getSeriesType(column).toString());
+ } catch (MetadataException e) {
+ throw new QueryProcessException(e);
+ }
}
}
@@ -856,11 +824,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
while (loc < totalSize) {
boolean isNonExist = false;
boolean isConstant = false;
- TSDataType type = null;
- String column = null;
+ TSDataType type;
+ String column;
// not exist
- if (notExistMeasurementsLoc < plan.getNotExistMeasurements().size()
- && loc == plan.getPositionOfNotExistMeasurements().get(notExistMeasurementsLoc)) {
+ if (isOneMeasurementIn(loc,
+ notExistMeasurementsLoc, plan.getPositionOfNotExistMeasurements())) {
// for shifting
plan.getPositionOfNotExistMeasurements().set(notExistMeasurementsLoc, loc - shiftLoc);
@@ -870,8 +838,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
isNonExist = true;
}
// constant
- else if (constMeasurementsLoc < plan.getConstMeasurements().size()
- && loc == plan.getPositionOfConstMeasurements().get(constMeasurementsLoc)) {
+ else if (isOneMeasurementIn(loc,
+ constMeasurementsLoc, plan.getPositionOfConstMeasurements())) {
// for shifting
plan.getPositionOfConstMeasurements().set(constMeasurementsLoc, loc - shiftLoc);
@@ -926,13 +894,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
plan.setDataTypeConsistencyChecker(null);
}
- private void getLastQueryHeaders(
- QueryPlan plan, List<String> respColumns, List<String> columnTypes)
- throws TException, QueryProcessException {
- respColumns.add(COLUMN_TIMESERIES);
- respColumns.add(COLUMN_VALUE);
- columnTypes.add(TSDataType.TEXT.toString());
- columnTypes.add(TSDataType.TEXT.toString());
+ /**
+ *
+ * @param subLoc
+ * @param totalLoc
+ * @param measurementPositions
+ * @return true if the measurement at totalLoc is the subLoc measurement in measurementPositions,
+ * false otherwise
+ */
+ private boolean isOneMeasurementIn(int totalLoc,
+ int subLoc, List<Integer> measurementPositions) {
+ return subLoc < measurementPositions.size() && totalLoc == measurementPositions.get(subLoc);
}
@Override
@@ -1066,12 +1038,16 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
IOException, MetadataException, SQLException {
- QueryContext context = new QueryContext(queryId);
+ QueryContext context = genQueryContext(queryId);
QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context);
queryId2DataSet.put(queryId, queryDataSet);
return queryDataSet;
}
+ protected QueryContext genQueryContext(long queryId) {
+ return new QueryContext(queryId);
+ }
+
@Override
public TSExecuteStatementResp executeUpdateStatement(TSExecuteStatementReq req) {
try {
@@ -1168,7 +1144,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
return resp;
}
- void handleClientExit() {
+ protected void handleClientExit() {
Long sessionId = currSessionId.get();
if (sessionId != null) {
TSCloseSessionReq req = new TSCloseSessionReq(sessionId);
@@ -1454,7 +1430,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
return null;
}
- private TSStatus executePlan(PhysicalPlan plan) {
+ protected TSStatus executePlan(PhysicalPlan plan) {
boolean execRet;
try {
execRet = executeNonQuery(plan);
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java b/server/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java
index 780603f..262de5f 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/IoTDBDataDirViewer.java
@@ -118,7 +118,7 @@ public class IoTDBDataDirViewer {
private static void printResource(String filename, PrintWriter pw) throws IOException {
filename = filename.substring(0, filename.length() - 9);
TsFileResource resource = new TsFileResource(SystemFileFactory.INSTANCE.getFile(filename));
- resource.deSerialize();
+ resource.deserialize();
// sort device strings
SortedSet<String> keys = new TreeSet<>(resource.getStartTimeMap().keySet());
for (String device : keys) {
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
index d52a220..03e97d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java
@@ -54,7 +54,7 @@ public class TsFileResourcePrinter {
filename = filename.substring(0, filename.length() - 9);
TsFileResource resource = new TsFileResource(SystemFileFactory.INSTANCE.getFile(filename));
System.err.println(String.format("analyzing %s ...", filename));
- resource.deSerialize();
+ resource.deserialize();
System.out.println("historicalVersions: " + resource.getHistoricalVersions());
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 7b93c94..765c17a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -53,7 +53,7 @@ public class FileLoaderUtils {
// write .resource file
tsFileResource.serialize();
} else {
- tsFileResource.deSerialize();
+ tsFileResource.deserialize();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index c18ce58..8ae97d8 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -18,15 +18,34 @@
*/
package org.apache.iotdb.db.utils;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PRIVILEGE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ROLE;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_STORAGE_GROUP;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TTL;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_USER;
+
+import java.util.Collections;
import java.util.List;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.constant.SQLConstant;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SchemaUtils {
+ private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
+
private SchemaUtils() {
}
@@ -48,11 +67,75 @@ public class SchemaUtils {
* @param schemaList the schema of the columns in this file.
* @return a Schema contains the provided schemas.
*/
- public static Schema getSchemaFromColumnSchema(List<MeasurementSchema> schemaList) {
+ private static Schema getSchemaFromColumnSchema(List<MeasurementSchema> schemaList) {
Schema schema = new Schema();
for (MeasurementSchema measurementSchema : schemaList) {
schema.registerMeasurement(measurementSchema);
}
return schema;
}
+
+ public static void registerTimeseries(MeasurementSchema schema) {
+ try {
+ logger.debug("Registering timeseries {}", schema);
+ String path = schema.getMeasurementId();
+ TSDataType dataType = schema.getType();
+ TSEncoding encoding = schema.getEncodingType();
+ CompressionType compressionType = schema.getCompressor();
+ boolean result = MManager.getInstance().createTimeseries(path, dataType, encoding,
+ compressionType, Collections.emptyMap());
+ if (result) {
+ StorageEngine.getInstance().addTimeSeries(new Path(path), dataType, encoding,
+ compressionType, Collections.emptyMap());
+ }
+ } catch (PathAlreadyExistException ignored) {
+ // ignore added timeseries
+ } catch (MetadataException | StorageEngineException e) {
+ logger.error("Cannot create timeseries {} in snapshot, ignored", schema.getMeasurementId(),
+ e);
+ }
+
+ }
+
+ public static TSDataType getSeriesType(String path)
+ throws MetadataException {
+ switch (path.toLowerCase()) {
+ // authorization queries
+ case COLUMN_ROLE:
+ case COLUMN_USER:
+ case COLUMN_PRIVILEGE:
+ case COLUMN_STORAGE_GROUP:
+ return TSDataType.TEXT;
+ case SQLConstant.RESERVED_TIME:
+ case COLUMN_TTL:
+ return TSDataType.INT64;
+ default:
+ // do nothing
+ }
+
+ if (path.contains("(") && !path.startsWith("(") && path.endsWith(")")) {
+ // aggregation
+ int leftBracketIndex = path.indexOf('(');
+ String aggrType = path.substring(0, leftBracketIndex);
+ String innerPath = path.substring(leftBracketIndex + 1, path.length() - 1);
+ switch (aggrType.toLowerCase()) {
+ case SQLConstant.MIN_TIME:
+ case SQLConstant.MAX_TIME:
+ case SQLConstant.COUNT:
+ return TSDataType.INT64;
+ case SQLConstant.LAST_VALUE:
+ case SQLConstant.FIRST_VALUE:
+ case SQLConstant.MIN_VALUE:
+ case SQLConstant.MAX_VALUE:
+ return getSeriesType(innerPath);
+ case SQLConstant.AVG:
+ case SQLConstant.SUM:
+ return TSDataType.DOUBLE;
+ default:
+ throw new MetadataException(
+ "aggregate does not support " + aggrType + " function.");
+ }
+ }
+ return MManager.getInstance().getSeriesType(path);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java b/server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java
index d4058ba..fa1acc2 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/TestOnly.java
@@ -27,5 +27,9 @@ import java.lang.annotation.Target;
@Target({ElementType.METHOD, ElementType.CONSTRUCTOR})
@Retention(RetentionPolicy.SOURCE)
+/**
+ * TestOnly implies that the method should only be used in the tests, otherwise its functionality
+ * is not guaranteed and may interfere with the normal code.
+ */
public @interface TestOnly {
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 941b494..bd0c2d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -150,7 +150,7 @@ public class TsFileRecoverPerformer {
private void recoverResourceFromFile() throws IOException {
try {
- tsFileResource.deSerialize();
+ tsFileResource.deserialize();
} catch (IOException e) {
logger.warn("Cannot deserialize TsFileResource {}, construct it using "
+ "TsFileSequenceReader", tsFileResource.getFile(), e);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java
index cf4bd93..4ad0991 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.junit.After;
@@ -45,7 +44,7 @@ public class MergeLogTest extends MergeTest {
File tempSGDir;
@Before
- public void setUp() throws IOException, WriteProcessException, MetadataException, PathException {
+ public void setUp() throws IOException, WriteProcessException, MetadataException {
super.setUp();
tempSGDir = new File(TestConstant.BASE_OUTPUT_PATH.concat("tempSG"));
tempSGDir.mkdirs();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
index a4b5e7c..656a215 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
@@ -26,9 +26,8 @@ import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
+import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -57,7 +56,7 @@ public class MergeOverLapTest extends MergeTest {
@Before
public void setUp()
- throws IOException, WriteProcessException, MetadataException, PathException {
+ throws IOException, WriteProcessException, MetadataException {
ptNum = 1000;
super.setUp();
tempSGDir = new File(TestConstant.BASE_OUTPUT_PATH.concat("tempSG"));
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
index e333c66..7b61725 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
@@ -25,10 +25,9 @@ import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -51,7 +50,7 @@ public class MergeTaskTest extends MergeTest {
private File tempSGDir;
@Before
- public void setUp() throws IOException, WriteProcessException, MetadataException, PathException {
+ public void setUp() throws IOException, WriteProcessException, MetadataException, MetadataException {
super.setUp();
tempSGDir = new File(TestConstant.BASE_OUTPUT_PATH.concat("tempSG"));
tempSGDir.mkdirs();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index 21a4683..028f0d0 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -34,7 +34,6 @@ import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.control.FileReaderManager;
@@ -71,7 +70,7 @@ abstract class MergeTest {
private int prevMergeChunkThreshold;
@Before
- public void setUp() throws IOException, WriteProcessException, MetadataException, PathException {
+ public void setUp() throws IOException, WriteProcessException, MetadataException, MetadataException {
MManager.getInstance().init();
prevMergeChunkThreshold =
IoTDBDescriptor.getInstance().getConfig().getChunkMergePointThreshold();
@@ -94,7 +93,7 @@ abstract class MergeTest {
MergeManager.getINSTANCE().stop();
}
- private void prepareSeries() throws MetadataException, PathException {
+ private void prepareSeries() throws MetadataException, MetadataException {
measurementSchemas = new MeasurementSchema[measurementNum];
for (int i = 0; i < measurementNum; i++) {
measurementSchemas[i] = new MeasurementSchema("sensor" + i, TSDataType.DOUBLE,
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 5be94ea..1ad0d22 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -35,19 +35,16 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
+import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.exception.storageGroup.StorageGroupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.reader.IPointReader;
@@ -232,7 +229,7 @@ public class DeletionFileNodeTest {
@Test
public void testDeleteInOverflowFile()
- throws StorageEngineException, QueryProcessException, IOException {
+ throws StorageEngineException, QueryProcessException {
// insert into BufferWrite
for (int i = 101; i <= 200; i++) {
TSRecord record = new TSRecord(i, processorName);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
index 9e4f16a..ce79ceb 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupException;
import org.apache.iotdb.db.metadata.MManager;
@@ -39,7 +38,6 @@ import org.apache.iotdb.db.qp.physical.crud.RawDataQueryPlan;
import org.apache.iotdb.db.query.executor.QueryRouter;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -66,8 +64,8 @@ public class DeletionQueryTest {
}
@Before
- public void setup() throws MetadataException,
- PathException, IOException, StorageEngineException, StartupException, StorageGroupException {
+ public void setup() throws
+ MetadataException, StorageEngineException, StartupException {
EnvironmentUtils.envSetUp();
MManager.getInstance().setStorageGroup(processorName);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FileNodeManagerBenchmark.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FileNodeManagerBenchmark.java
index 7fea61d..fc8d93a 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FileNodeManagerBenchmark.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/FileNodeManagerBenchmark.java
@@ -24,7 +24,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupException;
import org.apache.iotdb.db.metadata.MManager;
@@ -65,7 +64,7 @@ public class FileNodeManagerBenchmark {
}
private static void prepare()
- throws MetadataException, PathException, IOException, StorageGroupException {
+ throws MetadataException {
MManager manager = MManager.getInstance();
manager.setStorageGroup(prefix);
for (String device : devices) {
@@ -81,8 +80,8 @@ public class FileNodeManagerBenchmark {
}
public static void main(String[] args)
- throws InterruptedException, IOException, MetadataException,
- PathException, StorageEngineException, StorageGroupException {
+ throws InterruptedException, IOException,
+ MetadataException, StorageEngineException {
tearDown();
prepare();
long startTime = System.currentTimeMillis();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
index 832bde4..042a9fb 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
@@ -38,9 +38,7 @@ import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.OutOfTTLException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.exception.storageGroup.StorageGroupException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode;
@@ -77,7 +75,7 @@ public class TTLTest {
@Before
public void setUp()
- throws MetadataException, IOException, StartupException, PathException, StorageGroupProcessorException {
+ throws MetadataException, IOException, StartupException, StorageGroupProcessorException {
IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(86400);
EnvironmentUtils.envSetUp();
createSchemas();
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
index d0500f2..a10d9f3 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java
@@ -431,7 +431,7 @@ public class IoTDBMultiSeriesIT {
} catch (SQLException e) {
e.printStackTrace();
assertEquals(
- "401: Statement format is not right: Path [root.vehicle.d0.s10] does not exist",
+ "401: Statement format is not right: org.apache.iotdb.db.exception.metadata.PathNotExistException: Path [root.vehicle.d0.s10] does not exist",
e.getMessage());
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
index 7bcafa4..da1c011 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
@@ -26,7 +26,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
@@ -99,7 +98,7 @@ public class IoTDBSequenceDataQueryIT {
EnvironmentUtils.cleanEnv();
}
- private static void insertData() throws ClassNotFoundException, SQLException {
+ private static void insertData() throws ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection = DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
@@ -134,7 +133,7 @@ public class IoTDBSequenceDataQueryIT {
// insert data (time from 1200-1499)
for (long time = 1200; time < 1500; time++) {
- String sql = null;
+ String sql;
if (time % 2 == 0) {
sql = String
.format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, time % 17);
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index 4163e8a..c2cbd15 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@ -105,7 +105,7 @@ public class IoTDBSeriesReaderIT {
EnvironmentUtils.cleanEnv();
}
- private static void insertData() throws ClassNotFoundException, SQLException {
+ private static void insertData() throws ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection = DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTtlIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTtlIT.java
index 341627a..4fd5f13 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTtlIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTtlIT.java
@@ -142,7 +142,6 @@ public class IoTDBTtlIT {
Statement statement = connection.createStatement()) {
statement.execute("SET STORAGE GROUP TO root.group1");
statement.execute("SET STORAGE GROUP TO root.group2");
-
String result = doQuery(statement, "SHOW ALL TTL");
assertEquals("root.group1,null\n"
+ "root.group2,null\n", result);
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/plan/ConcatOptimizerTest.java b/server/src/test/java/org/apache/iotdb/db/qp/plan/ConcatOptimizerTest.java
index f6d6b67..015a094 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/plan/ConcatOptimizerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/plan/ConcatOptimizerTest.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
import org.antlr.v4.runtime.RecognitionException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
-import org.apache.iotdb.db.exception.query.PathException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.Planner;
@@ -49,7 +48,7 @@ public class ConcatOptimizerTest {
private Planner processor;
@Before
- public void before() throws MetadataException, PathException {
+ public void before() throws MetadataException {
processor = new Planner();
MManager.getInstance().init();
MManager.getInstance().setStorageGroup("root.laptop");
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java b/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java
index 1527685..1805579 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/plan/PhysicalPlanTest.java
@@ -93,8 +93,7 @@ public class PhysicalPlanTest {
String metadata = "create timeseries root.vehicle.d1.s2 with datatype=INT32,encoding=RLE";
Planner processor = new Planner();
CreateTimeSeriesPlan plan = (CreateTimeSeriesPlan) processor.parseSQLToPhysicalPlan(metadata);
- assertEquals(String.format("seriesPath: root.vehicle.d1.s2%n" + "resultDataType: INT32%n" +
- "encoding: RLE%nnamespace type: ADD_PATH%n" + "args: "), plan.toString());
+ assertEquals("seriesPath: root.vehicle.d1.s2, resultDataType: INT32, encoding: RLE, compression: SNAPPY", plan.toString());
}
@Test
@@ -102,8 +101,7 @@ public class PhysicalPlanTest {
String metadata = "create timeseries root.vehicle.d1.s2 with datatype=int32,encoding=rle";
Planner processor = new Planner();
CreateTimeSeriesPlan plan = (CreateTimeSeriesPlan) processor.parseSQLToPhysicalPlan(metadata);
- assertEquals(String.format("seriesPath: root.vehicle.d1.s2%n" + "resultDataType: INT32%n" +
- "encoding: RLE%nnamespace type: ADD_PATH%n" + "args: "), plan.toString());
+ assertEquals("seriesPath: root.vehicle.d1.s2, resultDataType: INT32, encoding: RLE, compression: SNAPPY", plan.toString());
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 5d7e025..d3df4a0 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -24,7 +24,6 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
-import java.net.SocketAddress;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
@@ -40,7 +39,6 @@ import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
-import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -187,12 +185,12 @@ public class EnvironmentUtils {
/**
* disable memory control</br> this function should be called before all code in the setup
*/
- public static void envSetUp() throws StartupException {
+ public static void envSetUp() {
logger.warn("EnvironmentUtil setup...");
System.setProperty(IoTDBConstant.REMOTE_JMX_PORT_NAME, "31999");
IoTDBDescriptor.getInstance().getConfig().setThriftServerAwaitTimeForStopService(0);
//we do not start 8181 port in test.
- IoTDBDescriptor.getInstance().getConfig().setEnableMetricsWebService(false);
+ IoTDBDescriptor.getInstance().getConfig().setEnableMetricService(false);
if (daemon == null) {
daemon = new IoTDB();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TimeValuePair.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TimeValuePair.java
index c52ee5b..035d3a5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TimeValuePair.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TimeValuePair.java
@@ -78,4 +78,5 @@ public class TimeValuePair implements Serializable, Comparable<TimeValuePair> {
public int compareTo(TimeValuePair o) {
return Long.compare(this.getTimestamp(), o.getTimestamp());
}
+
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
index eb483ac..a4c316f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
@@ -18,16 +18,20 @@
*/
package org.apache.iotdb.tsfile.read.common;
+import java.io.Serializable;
+import java.util.ArrayList;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.reader.BatchDataIterator;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
-import org.apache.iotdb.tsfile.utils.TsPrimitiveType.*;
-
-import java.io.Serializable;
-import java.util.ArrayList;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsBinary;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsBoolean;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsDouble;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsFloat;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsInt;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType.TsLong;
/**
* <code>BatchData</code> is a self-defined data structure which is optimized for different type of
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/TimeRange.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/TimeRange.java
index a3261c0..924a871 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/TimeRange.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/TimeRange.java
@@ -97,6 +97,10 @@ public class TimeRange implements Comparable<TimeRange> {
return min <= r.min && max >= r.max;
}
+ public boolean contains(long min, long max) {
+ return this.min <= min && this.max >= max;
+ }
+
/**
* Set a closed interval [min,max].
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
index e8cdb81..f2d82bf 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
@@ -146,4 +146,12 @@ public class GroupByFilter implements Filter, Serializable {
public int hashCode() {
return Objects.hash(interval, slidingStep, startTime, endTime);
}
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
index 1bb76d4..3767f01 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
@@ -118,4 +118,8 @@ public class In<T extends Comparable<T>> implements Filter {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.IN;
}
+
+ public Set<T> getValues() {
+ return values;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Murmur128Hash.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Murmur128Hash.java
index 2bb5118..9bc1149 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Murmur128Hash.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Murmur128Hash.java
@@ -18,34 +18,51 @@
*/
package org.apache.iotdb.tsfile.utils;
-import java.nio.ByteBuffer;
-
public class Murmur128Hash {
+ private Murmur128Hash() {
+ // util class
+ }
+
/**
- * get hashcode of value by seed
+ * get hashcode of value by seed
+ *
* @param value value
- * @param seed seend
+ * @param seed seed
* @return hashcode of value
*/
- public static int hash(String value, int seed){
- return (int) hash3_x64_128(ByteBuffer.wrap(value.getBytes()), 0, value.length(), seed);
+ public static int hash(String value, int seed) {
+ return (int) innerHash(value.getBytes(), 0, value.getBytes().length, seed);
+ }
+
+ /**
+ * get hashcode of two values by seed
+ *
+ * @param value1 the first value
+ * @param value2 the second value
+ * @param seed seed
+ * @return hashcode of value
+ */
+ public static int hash(String value1, long value2, int seed) {
+ return (int) innerHash(
+ BytesUtils.concatByteArray(value1.getBytes(), BytesUtils.longToBytes(value2)), 0,
+ value1.length() + 8, seed);
}
/**************************************
* Methods to perform murmur 128 hash.
**************************************/
- private static long getBlock(ByteBuffer key, int offset, int index) {
- int i_8 = index << 3;
- int blockOffset = offset + i_8;
- return ((long) key.get(blockOffset) & 0xff) + (((long) key.get(blockOffset + 1) & 0xff) << 8)
+ private static long getBlock(byte[] key, int offset, int index) {
+ int i8 = index << 3;
+ int blockOffset = offset + i8;
+ return ((long) key[blockOffset] & 0xff) + (((long) key[blockOffset + 1] & 0xff) << 8)
+
- (((long) key.get(blockOffset + 2) & 0xff) << 16) + (
- ((long) key.get(blockOffset + 3) & 0xff) << 24) +
- (((long) key.get(blockOffset + 4) & 0xff) << 32) + (
- ((long) key.get(blockOffset + 5) & 0xff) << 40) +
- (((long) key.get(blockOffset + 6) & 0xff) << 48) + (
- ((long) key.get(blockOffset + 7) & 0xff) << 56);
+ (((long) key[blockOffset + 2] & 0xff) << 16) + (
+ ((long) key[blockOffset + 3] & 0xff) << 24) +
+ (((long) key[blockOffset + 4] & 0xff) << 32) + (
+ ((long) key[blockOffset + 5] & 0xff) << 40) +
+ (((long) key[blockOffset + 6] & 0xff) << 48) + (
+ ((long) key[blockOffset + 7] & 0xff) << 56);
}
private static long rotl64(long v, int n) {
@@ -61,7 +78,7 @@ public class Murmur128Hash {
return k;
}
- private static long hash3_x64_128(ByteBuffer key, int offset, int length, long seed) {
+ private static long innerHash(byte[] key, int offset, int length, long seed) {
final int nblocks = length >> 4; // Process as 128-bit blocks.
long h1 = seed;
long h2 = seed;
@@ -70,7 +87,7 @@ public class Murmur128Hash {
// ----------
// body
for (int i = 0; i < nblocks; i++) {
- long k1 = getBlock(key, offset, i * 2 + 0);
+ long k1 = getBlock(key, offset, i * 2);
long k2 = getBlock(key, offset, i * 2 + 1);
k1 *= c1;
k1 = rotl64(k1, 31);
@@ -95,53 +112,53 @@ public class Murmur128Hash {
long k2 = 0;
switch (length & 15) {
case 15:
- k2 ^= ((long) key.get(offset + 14)) << 48;
+ k2 ^= ((long) key[offset + 14]) << 48;
// fallthrough
case 14:
- k2 ^= ((long) key.get(offset + 13)) << 40;
+ k2 ^= ((long) key[offset + 13]) << 40;
// fallthrough
case 13:
- k2 ^= ((long) key.get(offset + 12)) << 32;
+ k2 ^= ((long) key[offset + 12]) << 32;
// fallthrough
case 12:
- k2 ^= ((long) key.get(offset + 11)) << 24;
+ k2 ^= ((long) key[offset + 11]) << 24;
// fallthrough
case 11:
- k2 ^= ((long) key.get(offset + 10)) << 16;
+ k2 ^= ((long) key[offset + 10]) << 16;
// fallthrough
case 10:
- k2 ^= ((long) key.get(offset + 9)) << 8;
+ k2 ^= ((long) key[offset + 9]) << 8;
// fallthrough
case 9:
- k2 ^= ((long) key.get(offset + 8));
+ k2 ^= key[offset + 8];
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
// fallthrough
case 8:
- k1 ^= ((long) key.get(offset + 7)) << 56;
+ k1 ^= ((long) key[offset + 7]) << 56;
// fallthrough
case 7:
- k1 ^= ((long) key.get(offset + 6)) << 48;
+ k1 ^= ((long) key[offset + 6]) << 48;
// fallthrough
case 6:
- k1 ^= ((long) key.get(offset + 5)) << 40;
+ k1 ^= ((long) key[offset + 5]) << 40;
// fallthrough
case 5:
- k1 ^= ((long) key.get(offset + 4)) << 32;
+ k1 ^= ((long) key[offset + 4]) << 32;
// fallthrough
case 4:
- k1 ^= ((long) key.get(offset + 3)) << 24;
+ k1 ^= ((long) key[offset + 3]) << 24;
// fallthrough
case 3:
- k1 ^= ((long) key.get(offset + 2)) << 16;
+ k1 ^= ((long) key[offset + 2]) << 16;
// fallthrough
case 2:
- k1 ^= ((long) key.get(offset + 1)) << 8;
+ k1 ^= ((long) key[offset + 1]) << 8;
// fallthrough
case 1:
- k1 ^= (key.get(offset));
+ k1 ^= (key[offset]);
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
@@ -164,4 +181,5 @@ public class Murmur128Hash {
return h1 + h2;
}
+
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
index 9d89621..2e305d9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/ReadWriteIOUtils.java
@@ -25,6 +25,7 @@ import static org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.ClassSerializeId.DO
import static org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.ClassSerializeId.FLOAT;
import static org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.ClassSerializeId.INTEGER;
import static org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.ClassSerializeId.LONG;
+import static org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.ClassSerializeId.NULL;
import static org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.ClassSerializeId.STRING;
import java.io.DataOutputStream;
@@ -781,7 +782,7 @@ public class ReadWriteIOUtils {
}
enum ClassSerializeId {
- LONG, DOUBLE, INTEGER, FLOAT, BINARY, BOOLEAN, STRING
+ LONG, DOUBLE, INTEGER, FLOAT, BINARY, BOOLEAN, STRING, NULL
}
public static void writeObject(Object value, DataOutputStream outputStream) {
@@ -806,6 +807,8 @@ public class ReadWriteIOUtils {
} else if (value instanceof Boolean) {
outputStream.write(BOOLEAN.ordinal());
outputStream.write(((Boolean) value) ? 1 : 0);
+ } else if (value == null) {
+ outputStream.write(NULL.ordinal());
} else {
outputStream.write(STRING.ordinal());
byte[] bytes = value.toString().getBytes();
@@ -835,6 +838,8 @@ public class ReadWriteIOUtils {
byte[] bytes = new byte[length];
buffer.get(bytes);
return new Binary(bytes);
+ case NULL:
+ return null;
case STRING:
default:
length = buffer.getInt();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java
index b0c4a34..91d64f6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java
@@ -318,4 +318,5 @@ public class MeasurementSchema implements Comparable<MeasurementSchema>, Seriali
sc.addTail("]");
return sc.toString();
}
+
}