You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2022/04/23 04:05:15 UTC
[iotdb] 04/04: new deser
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch IOTDB-2962
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 36620fc229e8372d18c47ccef79caa01f0850d0a
Merge: 5d442299f3 ce064f429c
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Sat Apr 23 12:04:51 2022 +0800
new deser
Jenkinsfile | 2 +-
checkstyle.xml | 7 +-
.../iotdb/cluster/log/applier/DataLogApplier.java | 6 +-
.../iotdb/cluster/metadata/CSchemaProcessor.java | 12 +-
.../iotdb/cluster/query/ClusterPlanRouter.java | 5 +-
.../server/handlers/caller/LogCatchUpHandler.java | 4 +-
.../confignode1conf/iotdb-confignode.properties | 29 +
.../src/test/resources/confignode1conf/logback.xml | 140 +++
.../confignode2conf/iotdb-confignode.properties | 29 +
.../src/test/resources/confignode2conf/logback.xml | 140 +++
.../confignode3conf/iotdb-confignode.properties | 29 +
.../src/test/resources/confignode3conf/logback.xml | 140 +++
docs/Development/ContributeGuide.md | 16 +
.../Data-Modeling/SchemaRegion-rocksdb.md | 6 +-
docs/UserGuide/Operate-Metadata/Node.md | 45 +
docs/UserGuide/Operate-Metadata/Storage-Group.md | 60 ++
docs/UserGuide/Query-Data/Select-Expression.md | 143 +++
docs/zh/Development/ContributeGuide.md | 17 +
.../Data-Modeling/SchemaRegion-rocksdb.md | 6 +-
docs/zh/UserGuide/Operate-Metadata/Node.md | 45 +
.../zh/UserGuide/Operate-Metadata/Storage-Group.md | 60 ++
docs/zh/UserGuide/Query-Data/Select-Expression.md | 143 +++
.../iotdb/hadoop/tsfile/TSFMRReadExample.java | 5 +-
.../apache/iotdb/hadoop/tsfile/TSFInputFormat.java | 11 +-
.../iotdb/hadoop/tsfile/record/HDFSTSRecord.java | 8 +-
.../org/apache/iotdb/hive/TSFHiveInputFormat.java | 6 +-
.../org/apache/iotdb/hive/TsFileDeserializer.java | 10 +-
.../org/apache/iotdb/influxdb/IoTDBInfluxDB.java | 2 +-
.../db/integration/IoTDBUDTFBuiltinFunctionIT.java | 190 ++++
.../dropwizard/DropwizardMetricManager.java | 25 +-
.../iotdb/metrics/dropwizard/MetricName.java | 6 +-
.../reporter/DropwizardMetricsExporter.java | 10 +-
.../metrics/dropwizard/reporter/IoTDBReporter.java | 16 +-
.../org/apache/iotdb/metrics/MetricManager.java | 6 +-
.../micrometer/MicrometerMetricManager.java | 21 +-
.../iotdb/commons/concurrent/ThreadName.java | 3 +-
.../apache/iotdb/commons/service/ServiceType.java | 3 +-
pom.xml | 7 +
procedure/pom.xml | 93 ++
.../iotdb/procedure/CompletedProcedureCleaner.java | 88 ++
.../procedure/CompletedProcedureRetainer.java | 23 +-
.../apache/iotdb/procedure/InternalProcedure.java | 61 ++
.../java/org/apache/iotdb/procedure/Procedure.java | 895 ++++++++++++++++++
.../apache/iotdb/procedure/ProcedureExecutor.java | 995 +++++++++++++++++++++
.../apache/iotdb/procedure/ProcedureLockState.java | 18 +-
.../apache/iotdb/procedure/RootProcedureStack.java | 177 ++++
.../iotdb/procedure/StateMachineProcedure.java | 329 +++++++
.../apache/iotdb/procedure/StoppableThread.java | 45 +-
.../iotdb/procedure/TimeoutExecutorThread.java | 107 +++
.../iotdb/procedure/conf/ProcedureNodeConfig.java | 153 ++++
.../conf/ProcedureNodeConfigDescriptor.java | 219 +++++
.../procedure/conf/ProcedureNodeConstant.java | 26 +-
.../procedure/env/ClusterProcedureEnvironment.java | 19 +-
.../exception/ProcedureAbortedException.java | 20 +-
.../procedure/exception/ProcedureException.java | 30 +-
.../exception/ProcedureSuspendedException.java | 25 +-
.../exception/ProcedureTimeoutException.java | 18 +-
.../exception/ProcedureYieldException.java | 26 +-
.../scheduler/AbstractProcedureScheduler.java | 211 +++++
.../procedure/scheduler/ProcedureScheduler.java | 109 +++
.../scheduler/SimpleProcedureScheduler.java | 78 ++
.../iotdb/procedure/service/ProcedureNode.java | 98 ++
.../procedure/service/ProcedureNodeMBean.java | 16 +-
.../iotdb/procedure/service/ProcedureServer.java | 123 +++
.../service/ProcedureServerCommandLine.java | 52 ++
.../service/ProcedureServerProcessor.java | 77 ++
.../procedure/service/ProcedureServiceHanlder.java | 41 +-
.../iotdb/procedure/store/IProcedureStore.java | 28 +-
.../iotdb/procedure/store/ProcedureStore.java | 195 ++++
.../apache/iotdb/procedure/store/ProcedureWAL.java | 106 +++
.../apache/iotdb/procedure/NoopProcedureStore.java | 70 ++
.../org/apache/iotdb/procedure/TestLockRegime.java | 46 +
.../org/apache/iotdb/procedure/TestProcEnv.java | 57 ++
.../apache/iotdb/procedure/TestProcedureBase.java | 83 ++
.../iotdb/procedure/TestProcedureExecutor.java | 111 +++
.../apache/iotdb/procedure/TestSTMProcedure.java | 57 ++
.../iotdb/procedure/entity/IncProcedure.java | 57 ++
.../iotdb/procedure/entity/NoopProcedure.java | 38 +-
.../procedure/entity/SimpleLockProcedure.java | 80 ++
.../iotdb/procedure/entity/SimpleSTMProcedure.java | 97 ++
.../iotdb/procedure/entity/SleepProcedure.java | 46 +
.../iotdb/procedure/entity/StuckProcedure.java | 59 ++
.../iotdb/procedure/entity/StuckSTMProcedure.java | 112 +++
.../procedure/service/TestProcedureService.java | 87 ++
.../iotdb/procedure/store/TestProcedureStore.java | 111 +++
.../iotdb/procedure/util/ProcedureTestUtil.java | 66 ++
server/pom.xml | 4 +
.../resources/conf/schema-rocksdb.properties | 86 ++
.../io/LocalTextModificationAccessor.java | 7 +-
.../mtree/store/disk/CachedMNodeContainer.java | 15 +
.../mtree/store/disk/schemafile/RecordUtils.java | 6 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 8 +-
.../db/metadata/schemaregion/SchemaEngine.java | 23 +-
.../schemaregion/rocksdb/RSchemaConfLoader.java | 182 ++++
.../rocksdb/RSchemaReadWriteHandler.java | 27 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 130 ++-
.../iotdb/db/mpp/buffer/DataBlockManager.java | 24 +-
.../apache/iotdb/db/mpp/buffer/ISinkHandle.java | 20 +-
.../apache/iotdb/db/mpp/buffer/ISourceHandle.java | 10 +-
.../org/apache/iotdb/db/mpp/buffer/SinkHandle.java | 45 +-
.../apache/iotdb/db/mpp/buffer/SourceHandle.java | 19 +-
.../apache/iotdb/db/mpp/buffer/StubSinkHandle.java | 4 +-
.../iotdb/db/mpp/common/filter/InFilter.java | 8 +-
.../iotdb/db/mpp/common/header/DatasetHeader.java | 7 +-
.../iotdb/db/mpp/common/schematree/SchemaTree.java | 15 +
.../mpp/common/schematree/SchemaTreeVisitor.java | 6 +-
.../apache/iotdb/db/mpp/execution/Coordinator.java | 5 +-
.../apache/iotdb/db/mpp/execution/DataDriver.java | 189 +---
.../org/apache/iotdb/db/mpp/execution/Driver.java | 426 ++++++++-
.../iotdb/db/mpp/execution/DriverContext.java | 13 +-
.../db/mpp/execution/FragmentInstanceContext.java | 98 +-
.../mpp/execution/FragmentInstanceExecution.java | 66 +-
.../FragmentInstanceFailureListener.java} | 17 +-
.../db/mpp/execution/FragmentInstanceManager.java | 57 +-
.../db/mpp/execution/FragmentInstanceState.java | 2 +-
.../execution/FragmentInstanceStateMachine.java | 182 ++++
.../IDriver.java} | 22 +-
.../iotdb/db/mpp/execution/QueryExecution.java | 3 +-
.../iotdb/db/mpp/execution/SchemaDriver.java | 140 +--
.../scheduler/FixedRateFragInsStateTracker.java | 5 +-
.../org/apache/iotdb/db/mpp/operator/Operator.java | 4 +-
.../db/mpp/operator/process/LimitOperator.java | 4 +-
.../db/mpp/operator/process/TimeJoinOperator.java | 56 +-
.../db/mpp/operator/process/TransformOperator.java | 2 +-
.../mpp/operator/process/merge/ColumnMerger.java | 58 ++
.../operator/process/merge/SingleColumnMerger.java | 74 ++
.../mpp/operator/schema/SchemaFetchOperator.java | 3 +-
.../db/mpp/operator/source/ExchangeOperator.java | 2 +-
.../source/SeriesAggregateScanOperator.java | 6 +-
.../schedule/FragmentInstanceAbortedException.java | 4 +-
.../db/mpp/schedule/FragmentInstanceScheduler.java | 4 +-
.../mpp/schedule/FragmentInstanceTaskExecutor.java | 4 +-
.../mpp/schedule/IFragmentInstanceScheduler.java | 6 +-
.../db/mpp/schedule/task/FragmentInstanceTask.java | 16 +-
.../apache/iotdb/db/mpp/sql/analyze/Analyzer.java | 328 +++----
.../db/mpp/sql/analyze/FakeSchemaFetcherImpl.java | 7 +-
.../db/mpp/sql/parser/StatementGenerator.java | 20 +-
.../db/mpp/sql/planner/DistributionPlanner.java | 7 +-
.../{plan => }/IFragmentParallelPlaner.java | 4 +-
.../db/mpp/sql/planner/LocalExecutionPlanner.java | 28 +-
.../iotdb/db/mpp/sql/planner/LogicalPlanner.java | 5 +-
.../iotdb/db/mpp/sql/planner/QueryPlanBuilder.java | 10 +-
.../{plan => }/SimpleFragmentParallelPlanner.java | 5 +-
.../{plan => }/WriteFragmentParallelPlanner.java | 5 +-
.../db/mpp/sql/planner/plan/node/PlanNode.java | 8 +
.../db/mpp/sql/planner/plan/node/PlanNodeId.java | 16 -
.../node/metedata/read/DevicesSchemaScanNode.java | 17 +
.../plan/node/metedata/read/SchemaFetchNode.java | 17 +
.../plan/node/metedata/read/SchemaMergeNode.java | 17 +
.../metedata/read/TimeSeriesSchemaScanNode.java | 17 +
.../node/metedata/write/AlterTimeSeriesNode.java | 17 +
.../plan/node/metedata/write/AuthorNode.java | 17 +
.../write/CreateAlignedTimeSeriesNode.java | 16 +
.../node/metedata/write/CreateTimeSeriesNode.java | 16 +
.../planner/plan/node/process/AggregateNode.java | 31 +-
.../planner/plan/node/process/DeviceMergeNode.java | 53 +-
.../planner/plan/node/process/ExchangeNode.java | 17 +
.../sql/planner/plan/node/process/FillNode.java | 9 +-
.../sql/planner/plan/node/process/FilterNode.java | 10 +-
.../planner/plan/node/process/FilterNullNode.java | 81 +-
.../plan/node/process/GroupByLevelNode.java | 85 +-
.../sql/planner/plan/node/process/LimitNode.java | 9 +-
.../sql/planner/plan/node/process/OffsetNode.java | 9 +-
.../sql/planner/plan/node/process/SortNode.java | 11 +-
.../planner/plan/node/process/TimeJoinNode.java | 116 ++-
.../planner/plan/node/sink/FragmentSinkNode.java | 17 +
.../plan/node/source/SeriesAggregateScanNode.java | 37 +-
.../planner/plan/node/source/SeriesScanNode.java | 21 +-
.../plan/node/write/InsertMultiTabletsNode.java | 23 +-
.../sql/planner/plan/node/write/InsertNode.java | 66 ++
.../sql/planner/plan/node/write/InsertRowNode.java | 49 +-
.../planner/plan/node/write/InsertRowsNode.java | 17 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 17 +
.../planner/plan/node/write/InsertTabletNode.java | 57 +-
.../Aggregation.java} | 21 +-
.../planner/plan/parameter/AggregationStep.java | 73 ++
.../plan/parameter/FilterNullParameter.java | 88 ++
.../plan/parameter/GroupByTimeParameter.java} | 81 +-
.../plan/{ => parameter}/InputLocation.java | 35 +-
.../planner/plan/{ => parameter}/OutputColumn.java | 38 +-
.../db/mpp/sql/rewriter/ConcatPathRewriter.java | 9 +-
.../db/mpp/sql/rewriter/WildcardsRemover.java | 14 +-
.../component/GroupByLevelController.java | 8 +-
.../statement/component/GroupByTimeComponent.java | 73 --
.../statement/crud/AggregationQueryStatement.java | 7 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 8 +-
.../db/mpp/sql/statement/crud/QueryStatement.java | 17 +-
.../influxdb/function/InfluxFunctionFactory.java | 8 +-
.../protocol/influxdb/input/InfluxLineParser.java | 5 +-
.../protocol/influxdb/sql/InfluxDBSqlVisitor.java | 3 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 27 +-
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 9 +-
.../iotdb/db/query/executor/QueryRouter.java | 7 +-
.../db/query/executor/RawDataQueryExecutor.java | 7 +-
.../iotdb/db/query/expression/Expression.java | 75 +-
.../query/expression/unary/FunctionExpression.java | 3 +
.../db/query/factory/AggregateResultFactory.java | 17 +-
.../db/query/udf/builtin/BuiltinFunction.java | 5 +-
.../udf/builtin/UDTFEqualSizeBucketAggSample.java | 487 ++++++++++
.../udf/builtin/UDTFEqualSizeBucketM4Sample.java | 238 +++++
.../builtin/UDTFEqualSizeBucketRandomSample.java | 68 ++
.../udf/builtin/UDTFEqualSizeBucketSample.java | 48 +
.../thrift/impl/DataNodeTSIServiceImpl.java | 44 +-
.../transport/server/TransportServiceImpl.java | 9 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 6 +-
.../db/wal/recover/file/TsFilePlanRedoer.java | 19 +-
.../apache/iotdb/db/wal/utils/WALWriteUtils.java | 42 +
.../schemaregion/rocksdb/MRocksDBUnitTest.java | 29 +-
.../rocksdb/RSchemaRegionAdvancedTest.java | 4 +-
.../apache/iotdb/db/mpp/buffer/SinkHandleTest.java | 157 ++--
.../iotdb/db/mpp/buffer/SourceHandleTest.java | 93 +-
.../db/mpp/common/schematree/SchemaTreeTest.java | 14 +
.../db/mpp/execution/ConfigExecutionTest.java | 8 +-
.../iotdb/db/mpp/execution/DataDriverTest.java | 37 +-
.../iotdb/db/mpp/operator/LimitOperatorTest.java | 26 +-
.../operator/SeriesAggregateScanOperatorTest.java | 31 +-
.../db/mpp/operator/SeriesScanOperatorTest.java | 19 +-
.../db/mpp/operator/SingleColumnMergerTest.java | 117 +++
.../db/mpp/operator/TimeJoinOperatorTest.java | 144 ++-
.../operator/schema/SchemaScanOperatorTest.java | 32 +-
.../db/mpp/schedule/DefaultTaskSchedulerTest.java | 16 +-
.../schedule/FragmentInstanceSchedulerTest.java | 12 +-
.../FragmentInstanceTimeoutSentinelTest.java | 12 +-
.../db/mpp/sql/plan/FragmentInstanceSerdeTest.java | 9 +-
.../node/process/DeviceMergeNodeSerdeTest.java | 3 -
.../plan/node/process/ExchangeNodeSerdeTest.java | 3 -
.../sql/plan/node/process/FillNodeSerdeTest.java | 4 -
.../sql/plan/node/process/FilterNodeSerdeTest.java | 4 -
.../plan/node/process/FilterNullNodeSerdeTest.java | 9 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 9 +-
.../sql/plan/node/process/LimitNodeSerdeTest.java | 9 +-
.../sql/plan/node/process/OffsetNodeSerdeTest.java | 17 +-
.../sql/plan/node/process/SortNodeSerdeTest.java | 9 +-
.../plan/node/process/TimeJoinNodeSerdeTest.java | 13 +-
.../source/SeriesAggregateScanNodeSerdeTest.java | 6 +-
.../datanode1conf/iotdb-engine.properties | 37 +
.../test/resources/datanode1conf/iotdb-metric.yml | 21 +
.../src/test/resources/datanode1conf/logback.xml | 302 +++++++
.../datanode2conf/iotdb-engine.properties | 37 +
.../test/resources/datanode2conf/iotdb-metric.yml | 21 +
.../src/test/resources/datanode2conf/logback.xml | 302 +++++++
.../datanode3conf/iotdb-engine.properties | 37 +
.../test/resources/datanode3conf/iotdb-metric.yml | 21 +
.../src/test/resources/datanode3conf/logback.xml | 302 +++++++
thrift-procedure/pom.xml | 66 ++
.../src/main/thrift/procedure.thrift | 30 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 18 +-
.../apache/iotdb/tsfile/read/common/TimeRange.java | 24 +-
.../iotdb/tsfile/read/common/block/TsBlock.java | 10 +-
.../common/block/column/BinaryColumnBuilder.java | 19 +-
.../common/block/column/BooleanColumnBuilder.java | 19 +-
.../read/common/block/column/ColumnBuilder.java | 17 +-
.../common/block/column/DoubleColumnBuilder.java | 19 +-
.../common/block/column/FloatColumnBuilder.java | 15 +-
.../read/common/block/column/IntColumnBuilder.java | 19 +-
.../common/block/column/LongColumnBuilder.java | 19 +-
.../common/block/column/TimeColumnBuilder.java | 16 +-
.../read/controller/MetadataQuerierByFileImpl.java | 17 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 6 +-
.../tsfile/v2/read/TsFileSequenceReaderForV2.java | 17 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 4 -
.../tsfile/read/TsFileSequenceReaderTest.java | 47 +
262 files changed, 12670 insertions(+), 1754 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
index 8f689b50b1,1e481c2db5..4b588da3f5
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
@@@ -215,60 -201,7 +215,95 @@@ public abstract class Expression
}
}
- public void serialize(ByteBuffer byteBuffer) {
- ReadWriteIOUtils.write(isConstantOperandCache, byteBuffer);
+ protected abstract short getExpressionType();
+
+ public static void serialize(Expression expression, ByteBuffer byteBuffer) {
+ ReadWriteIOUtils.write(expression.getExpressionType(), byteBuffer);
++
+ expression.serialize(byteBuffer);
++
++ ReadWriteIOUtils.write(expression.inputColumnIndex != null, byteBuffer);
++ if (expression.inputColumnIndex != null) {
++ ReadWriteIOUtils.write(expression.inputColumnIndex, byteBuffer);
++ }
+ }
+
+ public abstract void serialize(ByteBuffer byteBuffer);
+
+ public static Expression deserialize(ByteBuffer byteBuffer) {
+ short type = ReadWriteIOUtils.readShort(byteBuffer);
++
++ Expression expression;
+ switch (type) {
+ case 0:
- return new AdditionExpression(byteBuffer);
++ expression = new AdditionExpression(byteBuffer);
++ break;
+ case 1:
- return new DivisionExpression(byteBuffer);
++ expression = new DivisionExpression(byteBuffer);
++ break;
+ case 2:
- return new EqualToExpression(byteBuffer);
++ expression = new EqualToExpression(byteBuffer);
++ break;
+ case 3:
- return new GreaterEqualExpression(byteBuffer);
++ expression = new GreaterEqualExpression(byteBuffer);
++ break;
+ case 4:
- return new GreaterThanExpression(byteBuffer);
++ expression = new GreaterThanExpression(byteBuffer);
++ break;
+ case 5:
- return new LessEqualExpression(byteBuffer);
++ expression = new LessEqualExpression(byteBuffer);
++ break;
+ case 6:
- return new LessThanExpression(byteBuffer);
++ expression = new LessThanExpression(byteBuffer);
++ break;
+ case 7:
- return new LogicAndExpression(byteBuffer);
++ expression = new LogicAndExpression(byteBuffer);
++ break;
+ case 8:
- return new LogicOrExpression(byteBuffer);
++ expression = new LogicOrExpression(byteBuffer);
++ break;
+ case 9:
- return new ModuloExpression(byteBuffer);
++ expression = new ModuloExpression(byteBuffer);
++ break;
+ case 10:
- return new MultiplicationExpression(byteBuffer);
++ expression = new MultiplicationExpression(byteBuffer);
++ break;
+ case 11:
- return new NonEqualExpression(byteBuffer);
++ expression = new NonEqualExpression(byteBuffer);
++ break;
+ case 12:
- return new SubtractionExpression(byteBuffer);
++ expression = new SubtractionExpression(byteBuffer);
++ break;
+ case 13:
- return new FunctionExpression(byteBuffer);
++ expression = new FunctionExpression(byteBuffer);
++ break;
+ case 14:
- return new LogicNotExpression(byteBuffer);
++ expression = new LogicNotExpression(byteBuffer);
++ break;
+ case 15:
- return new NegationExpression(byteBuffer);
++ expression = new NegationExpression(byteBuffer);
++ break;
+ case 16:
- return new TimeSeriesOperand(byteBuffer);
++ expression = new TimeSeriesOperand(byteBuffer);
++ break;
+ case 17:
- return new ConstantOperand(byteBuffer);
++ expression = new ConstantOperand(byteBuffer);
++ break;
+ case 18:
- throw new IllegalArgumentException("Invalid expression type: " + type);
++ expression = null;
++ break;
+ case 19:
- return new RegularExpression(byteBuffer);
++ expression = new RegularExpression(byteBuffer);
++ break;
+ default:
+ throw new IllegalArgumentException("Invalid expression type: " + type);
+ }
++
++ boolean hasInputColumnIndex = ReadWriteIOUtils.readBool(byteBuffer);
++ if (hasInputColumnIndex) {
++ expression.inputColumnIndex = ReadWriteIOUtils.readInt(byteBuffer);
++ }
++
++ return expression;
}
}