You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2020/02/28 06:57:17 UTC
[incubator-iotdb] 04/04: new TsFile
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch new_TsFile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 544453d77af0303f40f634c21c73e591bcd8a6d1
Merge: 3c7ea7b d6ae1c3
Author: HTHou <hh...@outlook.com>
AuthorDate: Fri Feb 28 14:51:07 2020 +0800
new TsFile
README.md | 2 +-
client-py/src/client_example.py | 33 +++++------
.../SystemDesign/5-DataQuery/2-SeriesReader.md | 2 +-
.../UserGuide/0-Get Started/1-QuickStart.md | 2 +-
.../UserGuide/4-Client/6-Status Codes.md | 65 +++++++++++++++++++++
.../2-DML (Data Manipulation Language).md | 23 +++++++-
.../5-Operation Manual/4-SQL Reference.md | 25 ++++----
docs/Documentation/UserGuide/0-Content.md | 1 +
.../UserGuide/0-Get Started/1-QuickStart.md | 9 ++-
.../UserGuide/4-Client/6-Status Codes.md | 66 ++++++++++++++++++++++
.../2-DML (Data Manipulation Language).md | 19 ++++++-
.../5-Operation Manual/4-SQL Reference.md | 12 ++--
.../main/java/org/apache/iotdb/JDBCExample.java | 2 +-
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 27 +++++----
.../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 7 +--
.../apache/iotdb/jdbc/IoTDBResultMetadataTest.java | 30 +++++-----
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 33 ++---------
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 33 +++++++----
.../apache/iotdb/db/engine/cache/ChunkCache.java | 37 +++++++-----
.../iotdb/db/engine/memtable/AbstractMemTable.java | 2 +-
.../iotdb/db/engine/merge/task/MergeTask.java | 3 +-
.../engine/storagegroup/StorageGroupProcessor.java | 37 +++++++-----
.../org/apache/iotdb/db/metadata/MManager.java | 2 +-
.../java/org/apache/iotdb/db/metadata/MTree.java | 64 +++++++++++++++++----
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 9 ++-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 13 +----
.../db/query/aggregation/impl/AvgAggrResult.java | 6 ++
.../db/query/aggregation/impl/CountAggrResult.java | 2 +-
.../iotdb/db/query/control/FileReaderManager.java | 34 +++++++----
.../iotdb/db/query/control/QueryFileManager.java | 17 +++---
.../dataset/RawQueryDataSetWithoutValueFilter.java | 13 +++--
.../iotdb/db/query/reader/series/SeriesReader.java | 8 +--
.../reader/universal/PriorityMergeReader.java | 26 +++++++--
.../org/apache/iotdb/db/service/TSServiceImpl.java | 19 ++++++-
.../java/org/apache/iotdb/db/utils/MemUtils.java | 21 ++++---
.../org/apache/iotdb/db/utils/SchemaUtils.java | 3 +-
.../integration/IOTDBGroupByInnerIntervalIT.java | 4 +-
.../iotdb/db/integration/IoTDBAggregationIT.java | 8 +--
.../integration/IoTDBAggregationSmallDataIT.java | 2 +-
.../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 4 +-
.../integration/IoTDBLoadExternalTsfileTest.java | 2 +-
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 4 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 4 ++
.../iotdb/db/integration/IoTDBSimpleQueryTest.java | 59 +++++++++++++++++++
.../iotdb/db/metadata/MManagerAdvancedTest.java | 14 +++--
.../iotdb/db/metadata/MManagerBasicTest.java | 57 ++++++++++++++++++-
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 16 ++++--
.../db/query/aggregation/AggregateResultTest.java | 2 +
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 +-
service-rpc/src/main/thrift/rpc.thrift | 3 +-
.../java/org/apache/iotdb/session/Session.java | 14 +++--
.../org/apache/iotdb/spark/db/Converter.scala | 37 ++++++------
.../org/apache/iotdb/spark/db/DefaultSource.scala | 37 ++++++------
.../org/apache/iotdb/spark/db/IoTDBOptions.scala | 37 ++++++------
.../scala/org/apache/iotdb/spark/db/IoTDBRDD.scala | 37 ++++++------
.../org/apache/iotdb/spark/db/IoTDBRelation.scala | 37 ++++++------
.../org/apache/iotdb/spark/db/Transformer.scala | 37 ++++++------
.../scala/org/apache/iotdb/spark/db/package.scala | 37 ++++++------
.../org/apache/iotdb/spark/db/IoTDBTest.scala | 37 ++++++------
.../org/apache/iotdb/spark/tsfile/Converter.scala | 37 ++++++------
.../apache/iotdb/spark/tsfile/DefaultSource.scala | 36 ++++++------
.../iotdb/spark/tsfile/NarrowConverter.scala | 37 ++++++------
.../spark/tsfile/NarrowTsFileOutputWriter.scala | 37 ++++++------
.../apache/iotdb/spark/tsfile/Transformer.scala | 37 ++++++------
.../iotdb/spark/tsfile/TsFileWriterFactory.scala | 36 ++++++------
.../apache/iotdb/spark/tsfile/WideConverter.scala | 37 ++++++------
.../spark/tsfile/WideTsFileOutputWriter.scala | 37 ++++++------
.../org/apache/iotdb/spark/tsfile/package.scala | 36 ++++++------
.../apache/iotdb/spark/tsfile/ConverterTest.scala | 37 ++++++------
.../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 37 ++++++------
.../iotdb/tsfile/read/TsFileSequenceReader.java | 27 ++++-----
.../tsfile/read/query/dataset/EmptyDataSet.java | 39 +++++++++++++
.../read/query/timegenerator/node/AndNode.java | 14 ++++-
.../read/query/timegenerator/node/LeafNode.java | 11 ++--
.../tsfile/read/reader/DefaultTsFileInput.java | 2 +-
.../tsfile/read/query/timegenerator/NodeTest.java | 66 +++++-----------------
.../iotdb/tsfile/read/reader/FakedBatchReader.java | 55 ++++++++++++++++++
78 files changed, 1172 insertions(+), 648 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
index b84b393,533bd9f..c17783f
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
@@@ -23,10 -23,7 +23,11 @@@ import java.io.File
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
++import java.util.Set;
import java.util.concurrent.Callable;
import org.apache.iotdb.db.engine.merge.manage.MergeContext;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
@@@ -113,15 -110,9 +114,15 @@@ public class MergeTask implements Calla
mergeLogger.logFiles(resource);
- List<String> devices = MManager.getInstance().getDevices(storageGroupName);
- List<MeasurementSchema> measurementSchemas = MManager.getInstance()
- .getStorageGroupSchema(storageGroupName);
- resource.addMeasurements(measurementSchemas);
++ Set<String> devices = MManager.getInstance().getDevices(storageGroupName);
+ Map<Path, MeasurementSchema> measurementSchemaMap = new HashMap<>();
+ for (String device : devices) {
+ Map<String, MeasurementSchema> schema = MManager.getInstance().getDeviceSchemaMap(device);
+ for (Entry<String, MeasurementSchema> entry : schema.entrySet()) {
+ measurementSchemaMap.put(new Path(device, entry.getKey()), entry.getValue());
+ }
+ }
+ resource.setMeasurementSchemaMap(measurementSchemaMap);
List<String> storageGroupPaths = MManager.getInstance().getAllTimeseriesName(storageGroupName + ".*");
List<Path> unmergedSeries = new ArrayList<>();
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 090590d,9a07c20..b8f8fd7
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@@ -538,15 -505,18 +540,18 @@@ public class MTree implements Serializa
return;
}
String nodeReg = MetaUtils.getNodeRegByIdx(idx, nodes);
- if (!(PATH_WILDCARD).equals(nodeReg)) {
+ if (!nodeReg.contains(PATH_WILDCARD)) {
if (node.hasChild(nodeReg)) {
findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
- timeseriesSchemaList);
+ MeasurementSchemaList);
}
} else {
for (MNode child : node.getChildren().values()) {
+ if (!Pattern.matches(nodeReg.replace("*", ".*"), child.getName())) {
+ continue;
+ }
findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR,
- timeseriesSchemaList);
+ MeasurementSchemaList);
}
}
}
diff --cc server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index 388dc13,c18ce58..21c99f6
--- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@@ -18,14 -18,9 +18,15 @@@
*/
package org.apache.iotdb.db.utils;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
++import java.util.Set;
+
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.Schema;
@@@ -42,15 -37,9 +43,15 @@@ public class SchemaUtils
* @return the schema of the FileNode named processorName.
*/
public static Schema constructSchema(String processorName) throws MetadataException {
- List<String> devices = MManager.getInstance().getDevices(processorName);
- List<MeasurementSchema> columnSchemaList;
- columnSchemaList = MManager.getInstance().getStorageGroupSchema(processorName);
- return getSchemaFromColumnSchema(columnSchemaList);
++ Set<String> devices = MManager.getInstance().getDevices(processorName);
+ Map<Path, MeasurementSchema> measurementSchemaMap = new HashMap<>();
+ for (String device : devices) {
+ Map<String, MeasurementSchema> schema = MManager.getInstance().getDeviceSchemaMap(device);
+ for (Entry<String, MeasurementSchema> entry : schema.entrySet()) {
+ measurementSchemaMap.put(new Path(device, entry.getKey()), entry.getValue());
+ }
+ }
+ return getSchemaFromColumnSchema(measurementSchemaMap);
}
/**
diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index a81a0b9,6fd2e43..8a3e96d
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@@ -53,6 -40,17 +41,19 @@@ import org.apache.iotdb.tsfile.write.sc
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+ import java.io.File;
+ import java.io.IOException;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.Comparator;
++import java.util.HashMap;
++import java.util.LinkedHashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.concurrent.ConcurrentHashMap;
+
+ import static org.apache.iotdb.tsfile.write.writer.TsFileIOWriter.magicStringBytes;
+
public class TsFileSequenceReader implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(TsFileSequenceReader.class);