You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/05/16 01:22:10 UTC
[iotdb] 01/04: opt
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch caLastOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dded9f0bf2acecabf3481c02a61764a9eabda482
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Thu May 11 21:52:30 2023 +0800
opt
---
.../db/engine/cache/TimeSeriesMetadataCache.java | 149 ++++++++++-----------
.../iotdb/db/mpp/execution/driver/Driver.java | 17 ++-
.../db/mpp/plan/parser/StatementGenerator.java | 7 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 3 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 104 +++++++-------
5 files changed, 128 insertions(+), 152 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 296e373dbfc..16cc83646d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -52,9 +52,6 @@ import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_CACHE;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_FILE;
-
/**
* This class is used to cache <code>TimeSeriesMetadata</code> in IoTDB. The caching strategy is
* LRU.
@@ -124,92 +121,86 @@ public class TimeSeriesMetadataCache {
throws IOException {
long startTime = System.nanoTime();
boolean cacheHit = true;
- try {
- if (!CACHE_ENABLE) {
- cacheHit = false;
-
- // bloom filter part
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
- BloomFilter bloomFilter = reader.readBloomFilter();
- if (bloomFilter != null
- && !bloomFilter.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
- return null;
- }
- TimeseriesMetadata timeseriesMetadata =
- reader.readTimeseriesMetadata(
- new Path(key.device, key.measurement, true), ignoreNotExists);
- return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0)
- ? null
- : timeseriesMetadata;
+ if (!CACHE_ENABLE) {
+ cacheHit = false;
+
+ // bloom filter part
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ BloomFilter bloomFilter = reader.readBloomFilter();
+ if (bloomFilter != null
+ && !bloomFilter.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
+ return null;
}
+ TimeseriesMetadata timeseriesMetadata =
+ reader.readTimeseriesMetadata(
+ new Path(key.device, key.measurement, true), ignoreNotExists);
+ return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0)
+ ? null
+ : timeseriesMetadata;
+ }
- TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
+ TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
- if (timeseriesMetadata == null) {
- if (debug) {
- DEBUG_LOGGER.info(
- "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
- DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
- }
- // allow for the parallelism of different devices
- synchronized (
- devices.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
- // double check
- timeseriesMetadata = lruCache.getIfPresent(key);
- if (timeseriesMetadata == null) {
- cacheHit = false;
-
- Path path = new Path(key.device, key.measurement, false);
- // bloom filter part
- BloomFilter bloomFilter =
- BloomFilterCache.getInstance()
- .get(new BloomFilterCache.BloomFilterCacheKey(key.filePath), debug);
- if (bloomFilter != null) {
- if (!bloomFilter.contains(path.getFullPath())) {
- if (debug) {
- DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
- }
- return null;
+ if (timeseriesMetadata == null) {
+ if (debug) {
+ DEBUG_LOGGER.info(
+ "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
+ DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
+ }
+ // allow for the parallelism of different devices
+ synchronized (
+ devices.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
+ // double check
+ timeseriesMetadata = lruCache.getIfPresent(key);
+ if (timeseriesMetadata == null) {
+ cacheHit = false;
+
+ Path path = new Path(key.device, key.measurement, false);
+ // bloom filter part
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(new BloomFilterCache.BloomFilterCacheKey(key.filePath), debug);
+ if (bloomFilter != null) {
+ if (!bloomFilter.contains(path.getFullPath())) {
+ if (debug) {
+ DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
+ return null;
}
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
- List<TimeseriesMetadata> timeSeriesMetadataList =
- reader.readTimeseriesMetadata(path, allSensors);
- // put TimeSeriesMetadata of all sensors used in this query into cache
- for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
- TimeSeriesMetadataCacheKey k =
- new TimeSeriesMetadataCacheKey(
- key.filePath, key.device, metadata.getMeasurementId());
- if (metadata.getStatistics().getCount() != 0) {
- lruCache.put(k, metadata);
- }
- if (metadata.getMeasurementId().equals(key.measurement)) {
- timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata;
- }
+ }
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ List<TimeseriesMetadata> timeSeriesMetadataList =
+ reader.readTimeseriesMetadata(path, allSensors);
+ // put TimeSeriesMetadata of all sensors used in this query into cache
+ for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
+ TimeSeriesMetadataCacheKey k =
+ new TimeSeriesMetadataCacheKey(
+ key.filePath, key.device, metadata.getMeasurementId());
+ if (metadata.getStatistics().getCount() != 0) {
+ lruCache.put(k, metadata);
+ }
+ if (metadata.getMeasurementId().equals(key.measurement)) {
+ timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata;
}
}
}
}
- if (timeseriesMetadata == null) {
- if (debug) {
- DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
- }
- return null;
- } else {
- if (debug) {
- DEBUG_LOGGER.info(
- "Get timeseries: {}.{} metadata in file: {} from cache: {}.",
- key.device,
- key.measurement,
- key.filePath,
- timeseriesMetadata);
- }
- return new TimeseriesMetadata(timeseriesMetadata);
+ }
+ if (timeseriesMetadata == null) {
+ if (debug) {
+ DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
+ }
+ return null;
+ } else {
+ if (debug) {
+ DEBUG_LOGGER.info(
+ "Get timeseries: {}.{} metadata in file: {} from cache: {}.",
+ key.device,
+ key.measurement,
+ key.filePath,
+ timeseriesMetadata);
}
- } finally {
- QUERY_METRICS.recordSeriesScanCost(
- cacheHit ? READ_TIMESERIES_METADATA_CACHE : READ_TIMESERIES_METADATA_FILE,
- System.nanoTime() - startTime);
+ return new TimeseriesMetadata(timeseriesMetadata);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
index a214522ca65..2ce330e9767 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
@@ -22,7 +22,6 @@ import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.execution.exchange.sink.ISink;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
-import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
import org.apache.iotdb.db.mpp.execution.schedule.task.DriverTaskId;
import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
@@ -383,14 +382,14 @@ public abstract class Driver implements IDriver {
sink.setNoMoreTsBlocks();
// record operator execution statistics to metrics
- List<OperatorContext> operatorContexts = driverContext.getOperatorContexts();
- for (OperatorContext operatorContext : operatorContexts) {
- String operatorType = operatorContext.getOperatorType();
- QUERY_METRICS.recordOperatorExecutionCost(
- operatorType, operatorContext.getTotalExecutionTimeInNanos());
- QUERY_METRICS.recordOperatorExecutionCount(
- operatorType, operatorContext.getNextCalledCount());
- }
+ // List<OperatorContext> operatorContexts = driverContext.getOperatorContexts();
+ // for (OperatorContext operatorContext : operatorContexts) {
+ // String operatorType = operatorContext.getOperatorType();
+ // QUERY_METRICS.recordOperatorExecutionCost(
+ // operatorType, operatorContext.getTotalExecutionTimeInNanos());
+ // QUERY_METRICS.recordOperatorExecutionCount(
+ // operatorType, operatorContext.getNextCalledCount());
+ // }
} catch (InterruptedException t) {
// don't record the stack
wasInterrupted = true;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index a4506737024..26b86e812ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@ -181,12 +181,7 @@ public class StatementGenerator {
// iterate the path list and add it to from operator
for (String pathStr : lastDataQueryReq.getPaths()) {
- PartialPath path;
- if (lastDataQueryReq.isLegalPathNodes()) {
- path = new PartialPath(pathStr.split("\\."));
- } else {
- path = new PartialPath(pathStr);
- }
+ PartialPath path = new PartialPath(pathStr.split("\\."));
fromComponent.addPrefixPath(path);
}
selectComponent.addResultColumn(
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
index 89cacc2361d..01bf9233621 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
@@ -437,7 +437,8 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
addStatementExecutionLatency(
OperationType.EXECUTE_LAST_DATA_QUERY,
StatementType.QUERY,
- System.nanoTime() - startNanoTime);
+ System.nanoTime() - startNanoTime,
+ TimeUnit.NANOSECONDS);
COORDINATOR.cleanupQueryExecution(queryId, t);
}
SESSION_MANAGER.updateIdleTime();
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index d558d0cf015..6ad023695e3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -53,8 +53,6 @@ import java.util.Map.Entry;
import java.util.Set;
import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_MEM;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_DISK;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_MEM;
import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_NONALIGNED;
public class FileLoaderUtils {
@@ -119,66 +117,58 @@ public class FileLoaderUtils {
throws IOException {
long t1 = System.nanoTime();
boolean loadFromMem = false;
- try {
- // common path
- TimeseriesMetadata timeSeriesMetadata;
- // If the tsfile is closed, we need to load from tsfile
- if (resource.isClosed()) {
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- timeSeriesMetadata =
- TimeSeriesMetadataCache.getInstance()
- .get(
- new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
- resource.getTsFilePath(),
- seriesPath.getDevice(),
- seriesPath.getMeasurement()),
- allSensors,
- resource.getTimeIndexType() != 1,
- context.isDebug());
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
- }
- } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
- loadFromMem = true;
-
- timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new MemChunkMetadataLoader(resource, seriesPath, context, filter));
- }
+ // common path
+ TimeseriesMetadata timeSeriesMetadata;
+ // If the tsfile is closed, we need to load from tsfile
+ if (resource.isClosed()) {
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ timeSeriesMetadata =
+ TimeSeriesMetadataCache.getInstance()
+ .get(
+ new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
+ resource.getTsFilePath(),
+ seriesPath.getDevice(),
+ seriesPath.getMeasurement()),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ context.isDebug());
+ if (timeSeriesMetadata != null) {
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
}
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
+ loadFromMem = true;
+ timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
if (timeSeriesMetadata != null) {
- long t2 = System.nanoTime();
- try {
- List<Modification> pathModifications =
- context.getPathModifications(resource.getModFile(), seriesPath);
- timeSeriesMetadata.setModified(!pathModifications.isEmpty());
- if (timeSeriesMetadata.getStatistics().getStartTime()
- > timeSeriesMetadata.getStatistics().getEndTime()) {
- return null;
- }
- if (filter != null
- && !filter.satisfyStartEndTime(
- timeSeriesMetadata.getStatistics().getStartTime(),
- timeSeriesMetadata.getStatistics().getEndTime())) {
- return null;
- }
- } finally {
- QUERY_METRICS.recordSeriesScanCost(
- TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2);
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new MemChunkMetadataLoader(resource, seriesPath, context, filter));
+ }
+ }
+
+ if (timeSeriesMetadata != null) {
+ long t2 = System.nanoTime();
+ try {
+ List<Modification> pathModifications =
+ context.getPathModifications(resource.getModFile(), seriesPath);
+ timeSeriesMetadata.setModified(!pathModifications.isEmpty());
+ if (timeSeriesMetadata.getStatistics().getStartTime()
+ > timeSeriesMetadata.getStatistics().getEndTime()) {
+ return null;
}
+ if (filter != null
+ && !filter.satisfyStartEndTime(
+ timeSeriesMetadata.getStatistics().getStartTime(),
+ timeSeriesMetadata.getStatistics().getEndTime())) {
+ return null;
+ }
+ } finally {
+ QUERY_METRICS.recordSeriesScanCost(
+ TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2);
}
- return timeSeriesMetadata;
- } finally {
- QUERY_METRICS.recordSeriesScanCost(
- loadFromMem
- ? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM
- : LOAD_TIMESERIES_METADATA_NONALIGNED_DISK,
- System.nanoTime() - t1);
}
+ return timeSeriesMetadata;
}
/**
@@ -278,7 +268,7 @@ public class FileLoaderUtils {
}
return alignedTimeSeriesMetadata;
} finally {
- if (loadFromMem) {
+ if (!loadFromMem) {
QUERY_METRICS.recordSeriesScanCost(
LOAD_TIMESERIES_METADATA_ALIGNED_MEM, System.nanoTime() - t1);
}