You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/06/20 06:21:08 UTC
[iotdb] branch ty/sonar updated: server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java done
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/sonar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ty/sonar by this push:
new f31bd373d43 server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java done
f31bd373d43 is described below
commit f31bd373d4301e849b18cc89a30c04dc2f0a6c47
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Jun 20 14:21:00 2023 +0800
server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java done
---
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 153 ++++++++++++---------
.../file/metadata/AlignedTimeSeriesMetadata.java | 3 +-
.../tsfile/file/metadata/ITimeSeriesMetadata.java | 3 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 6 +-
.../read/controller/IChunkMetadataLoader.java | 4 +-
5 files changed, 91 insertions(+), 78 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 8371584bb66..313386674d3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.iotdb.db.utils;
import org.apache.iotdb.commons.path.AlignedPath;
@@ -109,10 +110,13 @@ public class FileLoaderUtils {
}
/**
+ * Load TimeSeriesMetadata for non-aligned time series
+ *
* @param resource TsFile
* @param seriesPath Timeseries path
* @param allSensors measurements queried at the same time of this device
* @param filter any filter, only used to check time range
+ * @throws IOException IOException may be thrown while reading it from disk.
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public static TimeseriesMetadata loadTimeSeriesMetadata(
@@ -187,75 +191,35 @@ public class FileLoaderUtils {
}
/**
- * Load VectorTimeSeriesMetadata for Vector
+ * Load AlignedTimeSeriesMetadata for aligned time series.
*
* @param resource corresponding TsFileResource
- * @param vectorPath instance of VectorPartialPath, vector's full path, e.g. (root.sg1.d1.vector,
+ * @param alignedPath instance of VectorPartialPath, vector's full path, e.g. (root.sg1.d1.vector,
* [root.sg1.d1.vector.s1, root.sg1.d1.vector.s2])
+ * @throws IOException IOException may be thrown while reading it from disk.
*/
public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
- TsFileResource resource, AlignedPath vectorPath, QueryContext context, Filter filter)
+ TsFileResource resource, AlignedPath alignedPath, QueryContext context, Filter filter)
throws IOException {
- long t1 = System.nanoTime();
+ final long t1 = System.nanoTime();
boolean loadFromMem = false;
try {
- AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata;
// If the tsfile is closed, we need to load from tsfile
if (resource.isClosed()) {
- // load all the TimeseriesMetadata of vector, the first one is for time column and the
- // remaining is for sub sensors
- // the order of timeSeriesMetadata list is same as subSensorList's order
- TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
- List<String> valueMeasurementList = vectorPath.getMeasurementList();
- Set<String> allSensors = new HashSet<>(valueMeasurementList);
- allSensors.add("");
- boolean isDebug = context.isDebug();
- String filePath = resource.getTsFilePath();
- String deviceId = vectorPath.getDevice();
-
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- TimeseriesMetadata timeColumn =
- cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
- allSensors,
- resource.getTimeIndexType() != 1,
- isDebug);
- if (timeColumn != null) {
- List<TimeseriesMetadata> valueTimeSeriesMetadataList =
- new ArrayList<>(valueMeasurementList.size());
- // if all the queried aligned sensors does not exist, we will return null
- boolean exist = false;
- for (String valueMeasurement : valueMeasurementList) {
- TimeseriesMetadata valueColumn =
- cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
- allSensors,
- resource.getTimeIndexType() != 1,
- isDebug);
- exist = (exist || (valueColumn != null));
- valueTimeSeriesMetadataList.add(valueColumn);
- }
- if (exist) {
- alignedTimeSeriesMetadata =
- new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
- alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
- }
- }
+ alignedTimeSeriesMetadata = loadFromDisk(resource, alignedPath, context, filter);
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
loadFromMem = true;
-
alignedTimeSeriesMetadata =
- (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath);
+ (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(alignedPath);
if (alignedTimeSeriesMetadata != null) {
alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
+ new MemAlignedChunkMetadataLoader(resource, alignedPath, context, filter));
}
}
if (alignedTimeSeriesMetadata != null) {
- long t2 = System.nanoTime();
+ final long t2 = System.nanoTime();
try {
if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
> alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
@@ -269,19 +233,7 @@ public class FileLoaderUtils {
}
// set modifications to each aligned path
- List<TimeseriesMetadata> valueTimeSeriesMetadataList =
- alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
- boolean modified = false;
- for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
- if (valueTimeSeriesMetadataList.get(i) != null) {
- List<Modification> pathModifications =
- context.getPathModifications(
- resource.getModFile(), vectorPath.getPathWithMeasurement(i));
- valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
- modified = (modified || !pathModifications.isEmpty());
- }
- }
- alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
+ setModifications(resource, alignedTimeSeriesMetadata, alignedPath, context);
} finally {
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
TIMESERIES_METADATA_MODIFICATION_ALIGNED, System.nanoTime() - t2);
@@ -297,21 +249,90 @@ public class FileLoaderUtils {
}
}
+ private static AlignedTimeSeriesMetadata loadFromDisk(
+ TsFileResource resource, AlignedPath alignedPath, QueryContext context, Filter filter)
+ throws IOException {
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
+ // load all the TimeseriesMetadata of vector, the first one is for time column and the
+ // remaining is for sub sensors
+ // the order of timeSeriesMetadata list is same as subSensorList's order
+ TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+ List<String> valueMeasurementList = alignedPath.getMeasurementList();
+ Set<String> allSensors = new HashSet<>(valueMeasurementList);
+ allSensors.add("");
+ boolean isDebug = context.isDebug();
+ String filePath = resource.getTsFilePath();
+ String deviceId = alignedPath.getDevice();
+
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ TimeseriesMetadata timeColumn =
+ cache.get(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ isDebug);
+ if (timeColumn != null) {
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ new ArrayList<>(valueMeasurementList.size());
+ // if all the queried aligned sensors does not exist, we will return null
+ boolean exist = false;
+ for (String valueMeasurement : valueMeasurementList) {
+ TimeseriesMetadata valueColumn =
+ cache.get(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ isDebug);
+ exist = (exist || (valueColumn != null));
+ valueTimeSeriesMetadataList.add(valueColumn);
+ }
+ if (exist) {
+ alignedTimeSeriesMetadata =
+ new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
+ alignedTimeSeriesMetadata.setChunkMetadataLoader(
+ new DiskAlignedChunkMetadataLoader(resource, alignedPath, context, filter));
+ }
+ }
+ return alignedTimeSeriesMetadata;
+ }
+
+ private static void setModifications(
+ TsFileResource resource,
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata,
+ AlignedPath alignedPath,
+ QueryContext context) {
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
+ boolean modified = false;
+ for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
+ if (valueTimeSeriesMetadataList.get(i) != null) {
+ List<Modification> pathModifications =
+ context.getPathModifications(
+ resource.getModFile(), alignedPath.getPathWithMeasurement(i));
+ valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
+ modified = (modified || !pathModifications.isEmpty());
+ }
+ }
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
+ }
+
/**
* load all chunk metadata of one time series in one file.
*
* @param timeSeriesMetadata the corresponding TimeSeriesMetadata in that file.
*/
- public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata)
- throws IOException {
+ public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) {
return timeSeriesMetadata.loadChunkMetadataList();
}
/**
- * load all page readers in one chunk that satisfying the timeFilter
+ * load all page readers in one chunk that satisfying the timeFilter.
*
* @param chunkMetaData the corresponding chunk metadata
* @param timeFilter it should be a TimeFilter instead of a ValueFilter
+ * @throws IOException if chunkMetaData is null or errors happened while loading page readers,
+ * IOException will be thrown
*/
public static List<IPageReader> loadPageReaderList(
IChunkMetadata chunkMetaData, Filter timeFilter) throws IOException {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java
index f49d8142691..add0c7f5030 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -107,7 +106,7 @@ public class AlignedTimeSeriesMetadata implements ITimeSeriesMetadata {
* the subSensors to generate the VectorChunkMetadata
*/
@Override
- public List<IChunkMetadata> loadChunkMetadataList() throws IOException {
+ public List<IChunkMetadata> loadChunkMetadataList() {
return chunkMetadataLoader.loadChunkMetadataList(this);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java
index e7b85489460..6b6bbc31730 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.tsfile.file.metadata;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
-import java.io.IOException;
import java.util.List;
public interface ITimeSeriesMetadata {
@@ -37,7 +36,7 @@ public interface ITimeSeriesMetadata {
void setSeq(boolean seq);
/** @return the result has already been filtered by modification files */
- List<IChunkMetadata> loadChunkMetadataList() throws IOException;
+ List<IChunkMetadata> loadChunkMetadataList();
void setChunkMetadataLoader(IChunkMetadataLoader chunkMetadataLoader);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 2b010e25f6e..b2c98378ab1 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -230,12 +230,8 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata {
this.chunkMetadataLoader = chunkMetadataLoader;
}
- public IChunkMetadataLoader getChunkMetadataLoader() {
- return chunkMetadataLoader;
- }
-
@Override
- public List<IChunkMetadata> loadChunkMetadataList() throws IOException {
+ public List<IChunkMetadata> loadChunkMetadataList() {
return chunkMetadataLoader.loadChunkMetadataList(this);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
index fffe261bea7..5d8421637b4 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
@@ -21,12 +21,10 @@ package org.apache.iotdb.tsfile.read.controller;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
-import java.io.IOException;
import java.util.List;
public interface IChunkMetadataLoader {
/** read all chunk metadata of one time series in one file. */
- List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata)
- throws IOException;
+ List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata);
}