You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2020/12/29 07:03:38 UTC
[iotdb] branch cluster_multi_raft updated (72d697a -> ae1b6f3)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a change to branch cluster_multi_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
from 72d697a Merge branch 'master' into cluster_multi_raft
new c4a9dcd fix a bug of multi-raft
add 5fb992f Fix sync schema bug (#2309)
add 1ee1dd5 Update PartialPathTest.java (#2274)
add 1981766 remove useless coeds
add 852fbb9 Merge pull request #2302 from neuyilan/apache_master_1222_remove_useless_code
add 537aebf [IOTDB-944] Support UDTF (User-defined Timeseries Generating Function) (#1828)
add ac8c887 Update TsFileIOWriter.java (#2320)
add 21d356c Fix documents that are not linked to official websites (#2324)
add 92e26fb fix incorrect MetadataException printf info. (#2323)
add b4f41ba Remove a useless comma in the usage info of cluster (#2328)
add d76076d refactor the code
add 095803c Merge pull request #2339 from neuyilan/apache_master_1225_refactor_the_code
add 63c2aad fix wal buffer annotation (#2311)
add ec88e69 Bump maven wrapper from 3.6.1 to 3.6.3
add 231b334 Remove extra bracket to make the param annotation take effect
add 69de3d6 1. add pr template; 2. disable enforcer check in travis; 3. test sonar token
add 0b25cba supply the cluster version status code (#2342)
add 258d4ce Fix a potential overflow in udf (#2345)
add a3ea467 Fix the default value of operation timeout (#2330)
add a898e72 Delete update operation (#2288)
new fdc8d32 merge master
new ae1b6f3 assign slots
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/pull_request_template.md | 93 ++++
.github/workflows/sonar_and_coverall.yml | 2 +-
.mvn/wrapper/maven-wrapper.properties | 2 +-
.travis.yml | 10 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 52 +-
.../java/org/apache/iotdb/cluster/ClusterMain.java | 15 +-
.../cluster/client/async/AsyncClientPool.java | 1 -
.../apache/iotdb/cluster/config/ClusterConfig.java | 6 +-
.../apache/iotdb/cluster/log/LogDispatcher.java | 1 +
.../iotdb/cluster/log/catchup/LogCatchUpTask.java | 2 +
.../apache/iotdb/cluster/metadata/CMManager.java | 4 +-
.../iotdb/cluster/partition/PartitionGroup.java | 2 +-
.../cluster/query/ClusterConcatPathOptimizer.java | 4 +-
.../iotdb/cluster/query/ClusterPlanExecutor.java | 10 +-
.../iotdb/cluster/query/ClusterPlanRouter.java | 9 -
.../cluster/query/aggregate/ClusterAggregator.java | 1 +
.../cluster/query/reader/ClusterReaderFactory.java | 2 +
.../cluster/server/heartbeat/HeartbeatThread.java | 2 +
.../iotdb/cluster/server/member/RaftMember.java | 1 +
.../cluster/server/service/BaseSyncService.java | 7 +-
.../apache/iotdb/cluster/utils/ClusterUtils.java | 4 +-
.../org/apache/iotdb/cluster/common/IoTDBTest.java | 2 +-
.../cluster/log/applier/DataLogApplierTest.java | 4 +-
.../cluster/partition/SlotPartitionTableTest.java | 14 -
.../query/ClusterPhysicalGeneratorTest.java | 10 +-
.../cluster/query/ClusterPlanExecutorTest.java | 4 +-
docs/UserGuide/Client/Status Codes.md | 8 +
.../Operation Manual/UDF User Defined Function.md | 410 +++++++++++++++
docs/zh/UserGuide/Client/Status Codes.md | 9 +-
.../Operation Manual/UDF User Defined Function.md | 410 +++++++++++++++
example/pom.xml | 1 +
example/udf/pom.xml | 82 +++
.../java/org/apache/iotdb/udf/UDTFExample.java | 56 ++
.../resources/conf/iotdb-engine.properties | 24 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 15 +-
.../apache/iotdb/db/auth/entity/PrivilegeType.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 78 ++-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 3 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 43 ++
.../UDFRegistrationException.java} | 72 ++-
.../iotdb/db/metadata/logfile/MLogWriter.java | 38 +-
.../main/java/org/apache/iotdb/db/qp/Planner.java | 9 +-
.../apache/iotdb/db/qp/constant/DatetimeUtils.java | 40 ++
.../apache/iotdb/db/qp/constant/SQLConstant.java | 14 +-
.../apache/iotdb/db/qp/executor/IPlanExecutor.java | 11 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 70 ++-
.../org/apache/iotdb/db/qp/logical/Operator.java | 3 +-
.../iotdb/db/qp/logical/crud/SFWOperator.java | 16 +-
.../iotdb/db/qp/logical/crud/SelectOperator.java | 47 +-
.../CreateFunctionOperator.java} | 99 ++--
.../DropFunctionOperator.java} | 81 ++-
.../ShowFunctionsOperator.java} | 78 ++-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 2 +-
.../apache/iotdb/db/qp/physical/crud/UDFPlan.java | 79 +--
.../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 118 +++++
.../iotdb/db/qp/physical/crud/UpdatePlan.java | 135 -----
.../sys/{ShowPlan.java => CreateFunctionPlan.java} | 114 ++--
.../iotdb/db/qp/physical/sys/DropFunctionPlan.java | 81 +--
.../sys/ShowFunctionsPlan.java} | 75 ++-
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 6 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 152 ++++--
.../iotdb/db/qp/strategy/LogicalGenerator.java | 7 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 130 +++--
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 192 +++++--
.../db/query/aggregation/AggregateResult.java | 2 +-
.../db/query/aggregation/AggregationType.java | 1 -
.../db/query/control/QueryResourceManager.java | 6 +-
.../db/query/dataset/AlignByDeviceDataSet.java | 4 +-
.../db/query/dataset/DirectAlignByTimeDataSet.java | 72 ++-
.../db/query/dataset/DirectNonAlignDataSet.java | 70 ++-
.../apache/iotdb/db/query/dataset/ListDataSet.java | 4 +-
.../db/query/dataset/NonAlignEngineDataSet.java | 28 +-
.../dataset/RawQueryDataSetWithValueFilter.java | 4 +-
.../dataset/RawQueryDataSetWithoutValueFilter.java | 24 +-
.../db/query/dataset/ShowTimeseriesDataSet.java | 4 +-
.../db/query/dataset/UDTFAlignByTimeDataSet.java | 277 ++++++++++
.../apache/iotdb/db/query/dataset/UDTFDataSet.java | 152 ++++++
.../db/query/dataset/UDTFNonAlignDataSet.java | 191 +++++++
.../dataset/groupby/GroupByEngineDataSet.java | 4 +-
.../query/dataset/groupby/GroupByFillDataSet.java | 4 +-
.../query/dataset/groupby/GroupByTimeDataSet.java | 4 +-
.../groupby/GroupByWithValueFilterDataSet.java | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 2 +-
.../iotdb/db/query/executor/IQueryRouter.java | 8 +-
.../iotdb/db/query/executor/QueryRouter.java | 51 +-
.../db/query/executor/RawDataQueryExecutor.java | 22 +-
.../iotdb/db/query/executor/UDTFQueryExecutor.java | 84 +++
.../UpdateOperator.java => query/udf/api/UDF.java} | 70 ++-
.../org/apache/iotdb/db/query/udf/api/UDTF.java | 101 ++++
.../apache/iotdb/db/query/udf/api/access/Row.java | 119 +++++
.../iotdb/db/query/udf/api/access/RowIterator.java | 85 +--
.../iotdb/db/query/udf/api/access/RowWindow.java | 61 +++
.../db/query/udf/api/collector/PointCollector.java | 137 +++++
.../api/customizer/config/UDFConfigurations.java} | 79 ++-
.../api/customizer/config/UDTFConfigurations.java | 104 ++++
.../api/customizer/parameter/UDFParameters.java | 119 +++++
.../api/customizer/strategy/AccessStrategy.java | 64 +++
.../strategy/RowByRowAccessStrategy.java | 61 +++
.../strategy/SlidingSizeWindowAccessStrategy.java | 123 +++++
.../strategy/SlidingTimeWindowAccessStrategy.java | 249 +++++++++
.../iotdb/db/query/udf/core/access/RowImpl.java | 93 ++++
.../db/query/udf/core/access/RowIteratorImpl.java | 58 +++
.../db/query/udf/core/access/RowWindowImpl.java | 75 +++
.../db/query/udf/core/context/UDFContext.java | 134 +++++
.../db/query/udf/core/executor/UDTFExecutor.java | 92 ++++
.../iotdb/db/query/udf/core/input/InputLayer.java | 563 ++++++++++++++++++++
.../iotdb/db/query/udf/core/input/SafetyLine.java | 60 +++
.../db/query/udf/core/reader/LayerPointReader.java | 84 +--
.../udf/core/reader/LayerRowReader.java} | 79 ++-
.../udf/core/reader/LayerRowWindowReader.java} | 77 ++-
.../core/transformer/RawQueryPointTransformer.java | 72 +++
.../db/query/udf/core/transformer/Transformer.java | 93 ++++
.../core/transformer/UDFQueryRowTransformer.java | 84 +--
.../transformer/UDFQueryRowWindowTransformer.java | 46 ++
.../udf/core/transformer/UDFQueryTransformer.java | 89 ++++
.../query/udf/datastructure/SerializableList.java | 165 ++++++
.../primitive/ElasticSerializableIntList.java | 109 ++++
.../udf/datastructure/primitive/IntList.java} | 74 ++-
.../primitive/SerializableIntList.java | 131 +++++
.../datastructure/primitive/WrappedIntArray.java | 90 ++--
.../row/ElasticSerializableRowRecordList.java | 234 +++++++++
.../row/SerializableRowRecordList.java | 245 +++++++++
.../tv/ElasticSerializableBinaryTVList.java | 121 +++++
.../tv/ElasticSerializableTVList.java | 312 +++++++++++
.../datastructure/tv/SerializableBinaryTVList.java | 79 +++
.../tv/SerializableBooleanTVList.java | 69 +++
.../datastructure/tv/SerializableDoubleTVList.java | 69 +++
.../datastructure/tv/SerializableFloatTVList.java | 69 +++
.../datastructure/tv/SerializableIntTVList.java | 69 +++
.../datastructure/tv/SerializableLongTVList.java | 69 +++
.../udf/datastructure/tv/SerializableTVList.java | 99 ++++
.../udf/service/TemporaryQueryDataFileService.java | 136 +++++
.../iotdb/db/query/udf/service/UDFClassLoader.java | 64 +++
.../iotdb/db/query/udf/service/UDFLogWriter.java | 69 +++
.../udf/service/UDFRegistrationInformation.java} | 93 ++--
.../query/udf/service/UDFRegistrationService.java | 329 ++++++++++++
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 +
.../org/apache/iotdb/db/service/ServiceType.java | 4 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 102 ++--
.../db/sync/receiver/transfer/SyncServiceImpl.java | 20 +-
.../iotdb/db/sync/sender/transfer/SyncClient.java | 11 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 25 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 9 -
.../apache/iotdb/db/auth/AuthorityCheckerTest.java | 3 -
.../iotdb/db/engine/storagegroup/TTLTest.java | 2 +-
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 6 +-
.../iotdb/db/integration/IoTDBUDFManagementIT.java | 308 +++++++++++
.../db/integration/IoTDBUDFWindowQueryIT.java | 513 ++++++++++++++++++
.../integration/IoTDBUDTFAlignByTimeQueryIT.java | 525 +++++++++++++++++++
.../db/integration/IoTDBUDTFHybridQueryIT.java | 166 ++++++
.../db/integration/IoTDBUDTFNonAlignQueryIT.java | 337 ++++++++++++
.../apache/iotdb/db/metadata/PartialPathTest.java | 34 +-
.../iotdb/db/qp/plan/ConcatOptimizerTest.java | 7 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 213 +++++++-
.../query/dataset/UDTFAlignByTimeDataSetTest.java | 578 +++++++++++++++++++++
.../ElasticSerializableRowRecordListTest.java | 224 ++++++++
.../ElasticSerializableTVListTest.java | 251 +++++++++
.../SerializableBinaryTVListTest.java | 80 +++
.../SerializableBooleanTVListTest.java | 80 +++
.../SerializableDoubleTVListTest.java | 79 +++
.../datastructure/SerializableFloatTVListTest.java | 78 +++
.../datastructure/SerializableIntTVListTest.java | 78 +++
.../udf/datastructure/SerializableListTest.java} | 81 ++-
.../datastructure/SerializableLongTVListTest.java | 78 +++
.../SerializableRowRecordListTest.java | 143 +++++
.../udf/datastructure/SerializableTVListTest.java} | 79 ++-
.../iotdb/db/query/udf/example/Accumulator.java | 82 +++
.../apache/iotdb/db/query/udf/example/Adder.java | 78 +++
.../apache/iotdb/db/query/udf/example/Counter.java | 76 +++
.../iotdb/db/query/udf/example/Multiplier.java | 54 ++
.../SlidingSizeWindowConstructorTester0.java | 53 ++
.../SlidingSizeWindowConstructorTester1.java | 54 ++
.../SlidingTimeWindowConstructionTester.java | 61 +++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +
.../apache/iotdb/db/writelog/PerformanceTest.java | 15 +-
.../org/apache/iotdb/rpc/RpcTransportFactory.java | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 -
.../java/org/apache/iotdb/session/SessionUT.java | 2 +-
site/src/main/.vuepress/config.js | 26 +-
thrift/src/main/thrift/cluster.thrift | 17 +-
.../org/apache/iotdb/tsfile/read/common/Field.java | 4 -
.../apache/iotdb/tsfile/read/common/RowRecord.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 4 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/EmptyDataSet.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 4 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 11 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +-
188 files changed, 12971 insertions(+), 1568 deletions(-)
create mode 100644 .github/pull_request_template.md
create mode 100644 docs/UserGuide/Operation Manual/UDF User Defined Function.md
create mode 100644 docs/zh/UserGuide/Operation Manual/UDF User Defined Function.md
create mode 100644 example/udf/pom.xml
create mode 100644 example/udf/src/main/java/org/apache/iotdb/udf/UDTFExample.java
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => exception/UDFRegistrationException.java} (65%)
copy server/src/main/java/org/apache/iotdb/db/qp/logical/{crud/UpdateOperator.java => sys/CreateFunctionOperator.java} (51%)
copy server/src/main/java/org/apache/iotdb/db/qp/logical/{crud/UpdateOperator.java => sys/DropFunctionOperator.java} (66%)
copy server/src/main/java/org/apache/iotdb/db/qp/logical/{crud/UpdateOperator.java => sys/ShowFunctionsOperator.java} (68%)
copy cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterConcatPathOptimizer.java => server/src/main/java/org/apache/iotdb/db/qp/physical/crud/UDFPlan.java (57%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/qp/physical/crud/UDTFPlan.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/qp/physical/crud/UpdatePlan.java
copy server/src/main/java/org/apache/iotdb/db/qp/physical/sys/{ShowPlan.java => CreateFunctionPlan.java} (60%)
copy cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterConcatPathOptimizer.java => server/src/main/java/org/apache/iotdb/db/qp/physical/sys/DropFunctionPlan.java (61%)
copy server/src/main/java/org/apache/iotdb/db/qp/{logical/crud/UpdateOperator.java => physical/sys/ShowFunctionsPlan.java} (65%)
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/EmptyDataSet.java => server/src/main/java/org/apache/iotdb/db/query/dataset/DirectAlignByTimeDataSet.java (62%)
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/EmptyDataSet.java => server/src/main/java/org/apache/iotdb/db/query/dataset/DirectNonAlignDataSet.java (62%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/UDTFAlignByTimeDataSet.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/UDTFDataSet.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/UDTFNonAlignDataSet.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/executor/UDTFQueryExecutor.java
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => query/udf/api/UDF.java} (65%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/UDTF.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/access/Row.java
copy cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterConcatPathOptimizer.java => server/src/main/java/org/apache/iotdb/db/query/udf/api/access/RowIterator.java (50%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/access/RowWindow.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/collector/PointCollector.java
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => query/udf/api/customizer/config/UDFConfigurations.java} (60%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/customizer/config/UDTFConfigurations.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/customizer/parameter/UDFParameters.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/customizer/strategy/AccessStrategy.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/customizer/strategy/RowByRowAccessStrategy.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/customizer/strategy/SlidingSizeWindowAccessStrategy.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/api/customizer/strategy/SlidingTimeWindowAccessStrategy.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/access/RowImpl.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/access/RowIteratorImpl.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/access/RowWindowImpl.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/context/UDFContext.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/executor/UDTFExecutor.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/input/InputLayer.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/input/SafetyLine.java
copy cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterConcatPathOptimizer.java => server/src/main/java/org/apache/iotdb/db/query/udf/core/reader/LayerPointReader.java (51%)
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => query/udf/core/reader/LayerRowReader.java} (62%)
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => query/udf/core/reader/LayerRowWindowReader.java} (60%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/RawQueryPointTransformer.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/Transformer.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/EmptyDataSet.java => server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/UDFQueryRowTransformer.java (50%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/UDFQueryRowWindowTransformer.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/core/transformer/UDFQueryTransformer.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/SerializableList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/primitive/ElasticSerializableIntList.java
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => query/udf/datastructure/primitive/IntList.java} (65%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/primitive/SerializableIntList.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/EmptyDataSet.java => server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/primitive/WrappedIntArray.java (62%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/row/ElasticSerializableRowRecordList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/row/SerializableRowRecordList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/ElasticSerializableBinaryTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/ElasticSerializableTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableBinaryTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableBooleanTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableDoubleTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableFloatTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableIntTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableLongTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/datastructure/tv/SerializableTVList.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/service/TemporaryQueryDataFileService.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFClassLoader.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFLogWriter.java
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/crud/UpdateOperator.java => query/udf/service/UDFRegistrationInformation.java} (50%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationService.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDFManagementIT.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDFWindowQueryIT.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDTFAlignByTimeQueryIT.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDTFHybridQueryIT.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDTFNonAlignQueryIT.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/dataset/UDTFAlignByTimeDataSetTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/ElasticSerializableRowRecordListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/ElasticSerializableTVListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableBinaryTVListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableBooleanTVListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableDoubleTVListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableFloatTVListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableIntTVListTest.java
copy server/src/{main/java/org/apache/iotdb/db/qp/logical/crud/UpdateOperator.java => test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableListTest.java} (55%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableLongTVListTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableRowRecordListTest.java
rename server/src/{main/java/org/apache/iotdb/db/qp/logical/crud/UpdateOperator.java => test/java/org/apache/iotdb/db/query/udf/datastructure/SerializableTVListTest.java} (62%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/Accumulator.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/Adder.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/Counter.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/Multiplier.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/SlidingSizeWindowConstructorTester0.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/SlidingSizeWindowConstructorTester1.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/query/udf/example/SlidingTimeWindowConstructionTester.java
[iotdb] 03/03: assign slots
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster_multi_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae1b6f30341a36da9c773cf95a20e5d9c332b5e9
Author: lta <li...@163.com>
AuthorDate: Tue Dec 29 15:02:56 2020 +0800
assign slots
---
.../java/org/apache/iotdb/cluster/ClusterMain.java | 14 ++++----
.../iotdb/db/metadata/logfile/MLogWriter.java | 38 +++++++++++-----------
2 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java b/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java
index 491d577..bcde426 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java
@@ -288,8 +288,8 @@ public class ClusterMain {
@Override
public int calculateSlotByTime(String storageGroupName, long timestamp, int maxSlotNum) {
int sgSerialNum = extractSerialNumInSGName(storageGroupName) % k;
- if (sgSerialNum > 0) {
- return maxSlotNum / k * sgSerialNum;
+ if (sgSerialNum >= 0) {
+ return (int)(maxSlotNum / k * (sgSerialNum + 0.5));
} else {
return defaultStrategy.calculateSlotByTime(storageGroupName, timestamp, maxSlotNum);
}
@@ -299,20 +299,20 @@ public class ClusterMain {
public int calculateSlotByPartitionNum(String storageGroupName, long partitionId,
int maxSlotNum) {
int sgSerialNum = extractSerialNumInSGName(storageGroupName) % k;
- if (sgSerialNum > 0) {
- return maxSlotNum / k * sgSerialNum;
+ if (sgSerialNum >= 0) {
+ return (int)(maxSlotNum / k * (sgSerialNum + 0.5));
} else {
return defaultStrategy.calculateSlotByPartitionNum(storageGroupName, partitionId, maxSlotNum);
}
}
private int extractSerialNumInSGName(String storageGroupName) {
-// String[] s = storageGroupName.split("\\.");
- String[] s = storageGroupName.split("_");
+ String[] s = storageGroupName.split("\\.");
+// String[] s = storageGroupName.split("_");
if (s.length != 2) {
return -1;
}
-// s[1] = s[1].substring(4);
+ s[1] = s[1].substring(4);
try {
return Integer.parseInt(s[1]);
} catch (NumberFormatException e) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java
index 6191141..aa31a4e 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java
@@ -114,7 +114,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void createTimeseries(CreateTimeSeriesPlan createTimeSeriesPlan) throws IOException {
+ public synchronized void createTimeseries(CreateTimeSeriesPlan createTimeSeriesPlan) throws IOException {
try {
putLog(createTimeSeriesPlan);
} catch (BufferOverflowException e) {
@@ -123,7 +123,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void deleteTimeseries(DeleteTimeSeriesPlan deleteTimeSeriesPlan) throws IOException {
+ public synchronized void deleteTimeseries(DeleteTimeSeriesPlan deleteTimeSeriesPlan) throws IOException {
try {
putLog(deleteTimeSeriesPlan);
} catch (BufferOverflowException e) {
@@ -132,7 +132,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void setStorageGroup(PartialPath storageGroup) throws IOException {
+ public synchronized void setStorageGroup(PartialPath storageGroup) throws IOException {
try {
SetStorageGroupPlan plan = new SetStorageGroupPlan(storageGroup);
putLog(plan);
@@ -142,7 +142,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void deleteStorageGroup(PartialPath storageGroup) throws IOException {
+ public synchronized void deleteStorageGroup(PartialPath storageGroup) throws IOException {
try {
DeleteStorageGroupPlan plan = new DeleteStorageGroupPlan(Collections.singletonList(storageGroup));
putLog(plan);
@@ -152,7 +152,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void setTTL(PartialPath storageGroup, long ttl) throws IOException {
+ public synchronized void setTTL(PartialPath storageGroup, long ttl) throws IOException {
try {
SetTTLPlan plan = new SetTTLPlan(storageGroup, ttl);
putLog(plan);
@@ -162,7 +162,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void changeOffset(PartialPath path, long offset) throws IOException {
+ public synchronized void changeOffset(PartialPath path, long offset) throws IOException {
try {
ChangeTagOffsetPlan plan = new ChangeTagOffsetPlan(path, offset);
putLog(plan);
@@ -172,7 +172,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void changeAlias(PartialPath path, String alias) throws IOException {
+ public synchronized void changeAlias(PartialPath path, String alias) throws IOException {
try {
ChangeAliasPlan plan = new ChangeAliasPlan(path, alias);
putLog(plan);
@@ -182,7 +182,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void serializeMNode(MNode node) throws IOException {
+ public synchronized void serializeMNode(MNode node) throws IOException {
try {
int childSize = 0;
if (node.getChildren() != null) {
@@ -196,7 +196,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void serializeMeasurementMNode(MeasurementMNode node) throws IOException {
+ public synchronized void serializeMeasurementMNode(MeasurementMNode node) throws IOException {
try {
int childSize = 0;
if (node.getChildren() != null) {
@@ -211,7 +211,7 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void serializeStorageGroupMNode(StorageGroupMNode node) throws IOException {
+ public synchronized void serializeStorageGroupMNode(StorageGroupMNode node) throws IOException {
try {
int childSize = 0;
if (node.getChildren() != null) {
@@ -226,7 +226,7 @@ public class MLogWriter implements AutoCloseable {
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- public static void upgradeTxtToBin(String schemaDir, String oldFileName,
+ public synchronized static void upgradeTxtToBin(String schemaDir, String oldFileName,
String newFileName, boolean isSnapshot) throws IOException {
File logFile = SystemFileFactory.INSTANCE.getFile(schemaDir + File.separator + newFileName);
File tmpLogFile = SystemFileFactory.INSTANCE.getFile(logFile.getAbsolutePath() + ".tmp");
@@ -293,13 +293,13 @@ public class MLogWriter implements AutoCloseable {
}
}
- public static void upgradeMLog() throws IOException {
+ public synchronized static void upgradeMLog() throws IOException {
String schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
upgradeTxtToBin(schemaDir, MetadataConstant.METADATA_TXT_LOG, MetadataConstant.METADATA_LOG, false);
upgradeTxtToBin(schemaDir, MetadataConstant.MTREE_TXT_SNAPSHOT, MetadataConstant.MTREE_SNAPSHOT, true);
}
- public void clear() throws IOException {
+ public synchronized void clear() throws IOException {
sync();
logWriter.close();
mlogBuffer.clear();
@@ -310,18 +310,18 @@ public class MLogWriter implements AutoCloseable {
logWriter = new LogWriter(logFile, false);
}
- public int getLogNum() {
+ public synchronized int getLogNum() {
return logNum;
}
/**
* only used for initialize a mlog file writer.
*/
- public void setLogNum(int number) {
+ public synchronized void setLogNum(int number) {
logNum = number;
}
- public void operation(String cmd, boolean isSnapshot) throws IOException, MetadataException {
+ public synchronized void operation(String cmd, boolean isSnapshot) throws IOException, MetadataException {
if (!isSnapshot) {
operation(cmd);
} else {
@@ -344,7 +344,7 @@ public class MLogWriter implements AutoCloseable {
* @throws MetadataException
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- public void operation(String cmd) throws IOException, MetadataException {
+ public synchronized void operation(String cmd) throws IOException, MetadataException {
// see createTimeseries() to get the detailed format of the cmd
String[] args = cmd.trim().split(",", -1);
switch (args[0]) {
@@ -429,11 +429,11 @@ public class MLogWriter implements AutoCloseable {
}
}
- public void force() throws IOException {
+ public synchronized void force() throws IOException {
logWriter.force();
}
- public static PhysicalPlan convertFromString(String str) {
+ public synchronized static PhysicalPlan convertFromString(String str) {
String[] words = str.split(",");
switch (words[0]) {
case "2":
[iotdb] 01/03: fix a bug of multi-raft
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster_multi_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c4a9dcde75b8beba88bf84de2746a5cbf658b47c
Author: lta <li...@163.com>
AuthorDate: Mon Dec 28 17:31:28 2020 +0800
fix a bug of multi-raft
---
.../main/java/org/apache/iotdb/cluster/ClusterMain.java | 9 ++++++---
.../org/apache/iotdb/cluster/config/ClusterConfig.java | 2 +-
.../org/apache/iotdb/cluster/log/LogDispatcher.java | 1 +
.../iotdb/cluster/log/catchup/LogCatchUpTask.java | 2 ++
.../org/apache/iotdb/cluster/metadata/CMManager.java | 4 ++--
.../apache/iotdb/cluster/partition/PartitionGroup.java | 2 +-
.../cluster/query/aggregate/ClusterAggregator.java | 1 +
.../cluster/query/reader/ClusterReaderFactory.java | 2 ++
.../iotdb/cluster/server/heartbeat/HeartbeatThread.java | 2 ++
.../apache/iotdb/cluster/server/member/RaftMember.java | 1 +
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../java/org/apache/iotdb/rpc/RpcTransportFactory.java | 2 +-
thrift/src/main/thrift/cluster.thrift | 17 ++++++++---------
13 files changed, 29 insertions(+), 18 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java b/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java
index 14bedfd..3a67333 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java
@@ -282,7 +282,9 @@ public class ClusterMain {
// nodes evenly, and use default strategy for other groups
SlotPartitionTable.setSlotStrategy(new SlotStrategy() {
SlotStrategy defaultStrategy = new SlotStrategy.DefaultStrategy();
- int k = 2;
+ int k = ClusterDescriptor.getInstance().getConfig().getMultiRaftFactor() * ClusterDescriptor
+ .getInstance().getConfig().getSeedNodeUrls().size();
+
@Override
public int calculateSlotByTime(String storageGroupName, long timestamp, int maxSlotNum) {
int sgSerialNum = extractSerialNumInSGName(storageGroupName) % k;
@@ -305,11 +307,12 @@ public class ClusterMain {
}
private int extractSerialNumInSGName(String storageGroupName) {
- String[] s = storageGroupName.split("\\.");
+// String[] s = storageGroupName.split("\\.");
+ String[] s = storageGroupName.split("_");
if (s.length != 2) {
return -1;
}
- s[1] = s[1].substring(4);
+// s[1] = s[1].substring(4);
try {
return Integer.parseInt(s[1]);
} catch (NumberFormatException e) {
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
index 306b081..cc19a52 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
@@ -56,7 +56,7 @@ public class ClusterConfig {
private boolean useAsyncApplier = true;
- private int connectionTimeoutInMS = 20 * 1000;
+ private int connectionTimeoutInMS = 20_1000;
private int readOperationTimeoutMS = 30_1000;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
index 21820d5..9cab9fa 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
@@ -301,6 +301,7 @@ public class LogDispatcher {
private AppendEntriesRequest prepareRequest(List<ByteBuffer> logList,
List<SendLogRequest> currBatch, int firstIndex) {
AppendEntriesRequest request = new AppendEntriesRequest();
+ request.setRaftId(member.getRaftGroupId());
if (member.getHeader() != null) {
request.setHeader(member.getHeader());
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/catchup/LogCatchUpTask.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/catchup/LogCatchUpTask.java
index 8b69884..317e06f 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/catchup/LogCatchUpTask.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/catchup/LogCatchUpTask.java
@@ -86,6 +86,7 @@ public class LogCatchUpTask implements Callable<Boolean> {
void doLogCatchUp() throws TException, InterruptedException, LeaderUnknownException {
AppendEntryRequest request = new AppendEntryRequest();
+ request.setRaftId(raftId);
if (raftMember.getHeader() != null) {
request.setHeader(raftMember.getHeader());
}
@@ -170,6 +171,7 @@ public class LogCatchUpTask implements Callable<Boolean> {
private AppendEntriesRequest prepareRequest(List<ByteBuffer> logList, int startPos) {
AppendEntriesRequest request = new AppendEntriesRequest();
+ request.setRaftId(raftId);
if (raftMember.getHeader() != null) {
request.setHeader(raftMember.getHeader());
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
index df772a3..01817a4 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
@@ -938,8 +938,8 @@ public class CMManager extends MManager {
List<Node> coordinatedNodes = QueryCoordinator.getINSTANCE().reorderNodes(partitionGroup);
for (Node node : coordinatedNodes) {
try {
- List<PartialPath> paths = getMatchedPaths(node, partitionGroup.getHeader(), partitionGroup.getId(), pathsToQuery,
- withAlias);
+ List<PartialPath> paths = getMatchedPaths(node, partitionGroup.getHeader(),
+ partitionGroup.getId(), pathsToQuery, withAlias);
if (logger.isDebugEnabled()) {
logger.debug("{}: get matched paths of {} and other {} paths from {} in {}, result {}",
metaGroupMember.getName(), pathsToQuery.get(0), pathsToQuery.size() - 1, node,
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionGroup.java b/cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionGroup.java
index 5ab4275..2a562ac 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionGroup.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/partition/PartitionGroup.java
@@ -66,7 +66,7 @@ public class PartitionGroup extends ArrayList<Node> {
@Override
public int hashCode() {
- return Objects.hash(id, getHeader());
+ return Objects.hash(id, super.hashCode());
}
public Node getHeader() {
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/aggregate/ClusterAggregator.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/aggregate/ClusterAggregator.java
index c2b5974..6fb81b5 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/aggregate/ClusterAggregator.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/aggregate/ClusterAggregator.java
@@ -158,6 +158,7 @@ public class ClusterAggregator {
QueryContext context, boolean ascending) throws StorageEngineException {
GetAggrResultRequest request = new GetAggrResultRequest();
+ request.setRaftId(partitionGroup.getId());
request.setPath(path.getFullPath());
request.setAggregations(aggregations);
request.setDataTypeOrdinal(dataType.ordinal());
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
index d99bbbf..c5f6ee6 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
@@ -339,6 +339,7 @@ public class ClusterReaderFactory {
Set<String> deviceMeasurements, PartitionGroup partitionGroup,
QueryContext context, boolean ascending) {
SingleSeriesQueryRequest request = new SingleSeriesQueryRequest();
+ request.setRaftId(partitionGroup.getId());
if (timeFilter != null) {
request.setTimeFilterBytes(SerializeUtils.serializeFilter(timeFilter));
}
@@ -434,6 +435,7 @@ public class ClusterReaderFactory {
Set<String> deviceMeasurements, PartitionGroup partitionGroup,
QueryContext context, boolean ascending) throws StorageEngineException {
GroupByRequest request = new GroupByRequest();
+ request.setRaftId(partitionGroup.getId());
if (timeFilter != null) {
request.setTimeFilterBytes(SerializeUtils.serializeFilter(timeFilter));
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/heartbeat/HeartbeatThread.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/heartbeat/HeartbeatThread.java
index 0bfefee..2ccbaf3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/heartbeat/HeartbeatThread.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/heartbeat/HeartbeatThread.java
@@ -61,6 +61,7 @@ public class HeartbeatThread implements Runnable {
HeartbeatThread(RaftMember localMember) {
this.localMember = localMember;
memberName = localMember.getName();
+ request.setRaftId(localMember.getRaftGroupId());
}
@Override
@@ -202,6 +203,7 @@ public class HeartbeatThread implements Runnable {
req.setRequireIdentifier(request.requireIdentifier);
req.setTerm(request.term);
req.setLeader(localMember.getThisNode());
+ req.setRaftId(localMember.getRaftGroupId());
if (request.isSetHeader()) {
req.setHeader(request.header);
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
index cbf0fe4..4b4d4e1 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
@@ -1421,6 +1421,7 @@ public abstract class RaftMember {
AppendEntryRequest buildAppendEntryRequest(Log log, boolean serializeNow) {
AppendEntryRequest request = new AppendEntryRequest();
+ request.setRaftId(getRaftGroupId());
request.setTerm(term.get());
if (serializeNow) {
request.setEntry(log.serialize());
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 9847754..27af6b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -113,7 +113,7 @@ public class IoTDBConfig {
/**
* whether to use Snappy compression before sending data through the network
*/
- private boolean rpcAdvancedCompressionEnable = true;
+ private boolean rpcAdvancedCompressionEnable = false;
/**
* Port which the JDBC server listens to.
diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java
index 84f00df..57907de 100644
--- a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java
+++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcTransportFactory.java
@@ -27,7 +27,7 @@ import org.apache.thrift.transport.TTransportFactory;
public class RpcTransportFactory extends TTransportFactory {
// TODO: make it a config
- public static boolean USE_SNAPPY = true;
+ public static boolean USE_SNAPPY = false;
public static final RpcTransportFactory INSTANCE;
static {
INSTANCE = USE_SNAPPY ?
diff --git a/thrift/src/main/thrift/cluster.thrift b/thrift/src/main/thrift/cluster.thrift
index 549cd55..69a4dc5 100644
--- a/thrift/src/main/thrift/cluster.thrift
+++ b/thrift/src/main/thrift/cluster.thrift
@@ -41,7 +41,7 @@ struct HeartBeatRequest {
// because a data server may play many data groups members, this is used to identify which
// member should process the request or response. Only used in data group communication.
8: optional Node header
- 9: optional int raftId
+ 9: required int raftId
}
// follower -> leader
@@ -57,7 +57,6 @@ struct HeartBeatResponse {
// because a data server may play many data groups members, this is used to identify which
// member should process the request or response. Only used in data group communication.
7: optional Node header
- 8: optional int raftId
}
// node -> node
@@ -70,7 +69,7 @@ struct ElectionRequest {
// because a data server may play many data groups members, this is used to identify which
// member should process the request or response. Only used in data group communication.
5: optional Node header
- 6: optional int raftId
+ 6: required int raftId
7: optional long dataLogLastIndex
8: optional long dataLogLastTerm
}
@@ -87,7 +86,7 @@ struct AppendEntryRequest {
// because a data server may play many data groups members, this is used to identify which
// member should process the request or response. Only used in data group communication.
7: optional Node header
- 8: optional int raftId
+ 8: required int raftId
}
// leader -> follower
@@ -102,7 +101,7 @@ struct AppendEntriesRequest {
// because a data server may play many data groups members, this is used to identify which
// member should process the request or response. Only used in data group communication.
7: optional Node header
- 8: optional int raftId
+ 8: required int raftId
}
struct AddNodeResponse {
@@ -148,14 +147,14 @@ struct SendSnapshotRequest {
1: required binary snapshotBytes
// for data group
2: optional Node header
- 3: optional int raftId
+ 3: required int raftId
}
struct PullSnapshotRequest {
1: required list<int> requiredSlots
// for data group
2: optional Node header
- 3: optional int raftId
+ 3: required int raftId
// set to true if the previous holder has been removed from the cluster.
// This will make the previous holder read-only so that different new
// replicas can pull the same snapshot.
@@ -169,13 +168,13 @@ struct PullSnapshotResp {
struct ExecutNonQueryReq {
1: required binary planBytes
2: optional Node header
- 3: optional int raftId
+ 3: required int raftId
}
struct PullSchemaRequest {
1: required list<string> prefixPaths
2: optional Node header
- 3: optional int raftId
+ 3: required int raftId
}
struct PullSchemaResp {
[iotdb] 02/03: merge master
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster_multi_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fdc8d327c6d196a0ab94043f0e6913f3bd5f00f1
Merge: c4a9dcd a898e72
Author: lta <li...@163.com>
AuthorDate: Mon Dec 28 17:33:12 2020 +0800
merge master
.github/pull_request_template.md | 93 ++++
.github/workflows/sonar_and_coverall.yml | 2 +-
.mvn/wrapper/maven-wrapper.properties | 2 +-
.travis.yml | 10 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 52 +-
.../java/org/apache/iotdb/cluster/ClusterMain.java | 2 +-
.../cluster/client/async/AsyncClientPool.java | 1 -
.../apache/iotdb/cluster/config/ClusterConfig.java | 4 +-
.../cluster/query/ClusterConcatPathOptimizer.java | 4 +-
.../iotdb/cluster/query/ClusterPlanExecutor.java | 10 +-
.../iotdb/cluster/query/ClusterPlanRouter.java | 9 -
.../cluster/server/service/BaseSyncService.java | 7 +-
.../apache/iotdb/cluster/utils/ClusterUtils.java | 4 +-
.../org/apache/iotdb/cluster/common/IoTDBTest.java | 2 +-
.../cluster/log/applier/DataLogApplierTest.java | 4 +-
.../cluster/partition/SlotPartitionTableTest.java | 14 -
.../query/ClusterPhysicalGeneratorTest.java | 10 +-
.../cluster/query/ClusterPlanExecutorTest.java | 4 +-
docs/UserGuide/Client/Status Codes.md | 8 +
.../Operation Manual/UDF User Defined Function.md | 410 +++++++++++++++
docs/zh/UserGuide/Client/Status Codes.md | 9 +-
.../Operation Manual/UDF User Defined Function.md | 410 +++++++++++++++
example/pom.xml | 1 +
example/udf/pom.xml | 82 +++
.../java/org/apache/iotdb/udf/UDTFExample.java | 56 ++
.../resources/conf/iotdb-engine.properties | 24 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 15 +-
.../apache/iotdb/db/auth/entity/PrivilegeType.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 76 ++-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 3 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 43 ++
.../UDFRegistrationException.java} | 72 ++-
.../main/java/org/apache/iotdb/db/qp/Planner.java | 9 +-
.../apache/iotdb/db/qp/constant/DatetimeUtils.java | 40 ++
.../apache/iotdb/db/qp/constant/SQLConstant.java | 14 +-
.../apache/iotdb/db/qp/executor/IPlanExecutor.java | 11 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 70 ++-
.../org/apache/iotdb/db/qp/logical/Operator.java | 3 +-
.../iotdb/db/qp/logical/crud/SFWOperator.java | 16 +-
.../iotdb/db/qp/logical/crud/SelectOperator.java | 47 +-
.../CreateFunctionOperator.java} | 99 ++--
.../DropFunctionOperator.java} | 81 ++-
.../ShowFunctionsOperator.java} | 78 ++-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 2 +-
.../apache/iotdb/db/qp/physical/crud/UDFPlan.java | 79 +--
.../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 118 +++++
.../iotdb/db/qp/physical/crud/UpdatePlan.java | 135 -----
.../sys/{ShowPlan.java => CreateFunctionPlan.java} | 114 ++--
.../iotdb/db/qp/physical/sys/DropFunctionPlan.java | 81 +--
.../sys/ShowFunctionsPlan.java} | 75 ++-
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 6 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 152 ++++--
.../iotdb/db/qp/strategy/LogicalGenerator.java | 7 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 130 +++--
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 192 +++++--
.../db/query/aggregation/AggregateResult.java | 2 +-
.../db/query/aggregation/AggregationType.java | 1 -
.../db/query/control/QueryResourceManager.java | 6 +-
.../db/query/dataset/AlignByDeviceDataSet.java | 4 +-
.../db/query/dataset/DirectAlignByTimeDataSet.java | 72 ++-
.../db/query/dataset/DirectNonAlignDataSet.java | 70 ++-
.../apache/iotdb/db/query/dataset/ListDataSet.java | 4 +-
.../db/query/dataset/NonAlignEngineDataSet.java | 28 +-
.../dataset/RawQueryDataSetWithValueFilter.java | 4 +-
.../dataset/RawQueryDataSetWithoutValueFilter.java | 24 +-
.../db/query/dataset/ShowTimeseriesDataSet.java | 4 +-
.../db/query/dataset/UDTFAlignByTimeDataSet.java | 277 ++++++++++
.../apache/iotdb/db/query/dataset/UDTFDataSet.java | 152 ++++++
.../db/query/dataset/UDTFNonAlignDataSet.java | 191 +++++++
.../dataset/groupby/GroupByEngineDataSet.java | 4 +-
.../query/dataset/groupby/GroupByFillDataSet.java | 4 +-
.../query/dataset/groupby/GroupByTimeDataSet.java | 4 +-
.../groupby/GroupByWithValueFilterDataSet.java | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 2 +-
.../iotdb/db/query/executor/IQueryRouter.java | 8 +-
.../iotdb/db/query/executor/QueryRouter.java | 51 +-
.../db/query/executor/RawDataQueryExecutor.java | 22 +-
.../iotdb/db/query/executor/UDTFQueryExecutor.java | 84 +++
.../UpdateOperator.java => query/udf/api/UDF.java} | 70 ++-
.../org/apache/iotdb/db/query/udf/api/UDTF.java | 101 ++++
.../apache/iotdb/db/query/udf/api/access/Row.java | 119 +++++
.../iotdb/db/query/udf/api/access/RowIterator.java | 85 +--
.../iotdb/db/query/udf/api/access/RowWindow.java | 61 +++
.../db/query/udf/api/collector/PointCollector.java | 137 +++++
.../api/customizer/config/UDFConfigurations.java} | 79 ++-
.../api/customizer/config/UDTFConfigurations.java | 104 ++++
.../api/customizer/parameter/UDFParameters.java | 119 +++++
.../api/customizer/strategy/AccessStrategy.java | 64 +++
.../strategy/RowByRowAccessStrategy.java | 61 +++
.../strategy/SlidingSizeWindowAccessStrategy.java | 123 +++++
.../strategy/SlidingTimeWindowAccessStrategy.java | 249 +++++++++
.../iotdb/db/query/udf/core/access/RowImpl.java | 93 ++++
.../db/query/udf/core/access/RowIteratorImpl.java | 58 +++
.../db/query/udf/core/access/RowWindowImpl.java | 75 +++
.../db/query/udf/core/context/UDFContext.java | 134 +++++
.../db/query/udf/core/executor/UDTFExecutor.java | 92 ++++
.../iotdb/db/query/udf/core/input/InputLayer.java | 563 ++++++++++++++++++++
.../iotdb/db/query/udf/core/input/SafetyLine.java | 60 +++
.../db/query/udf/core/reader/LayerPointReader.java | 84 +--
.../udf/core/reader/LayerRowReader.java} | 79 ++-
.../udf/core/reader/LayerRowWindowReader.java} | 77 ++-
.../core/transformer/RawQueryPointTransformer.java | 72 +++
.../db/query/udf/core/transformer/Transformer.java | 93 ++++
.../core/transformer/UDFQueryRowTransformer.java | 84 +--
.../transformer/UDFQueryRowWindowTransformer.java | 46 ++
.../udf/core/transformer/UDFQueryTransformer.java | 89 ++++
.../query/udf/datastructure/SerializableList.java | 165 ++++++
.../primitive/ElasticSerializableIntList.java | 109 ++++
.../udf/datastructure/primitive/IntList.java} | 74 ++-
.../primitive/SerializableIntList.java | 131 +++++
.../datastructure/primitive/WrappedIntArray.java | 90 ++--
.../row/ElasticSerializableRowRecordList.java | 234 +++++++++
.../row/SerializableRowRecordList.java | 245 +++++++++
.../tv/ElasticSerializableBinaryTVList.java | 121 +++++
.../tv/ElasticSerializableTVList.java | 312 +++++++++++
.../datastructure/tv/SerializableBinaryTVList.java | 79 +++
.../tv/SerializableBooleanTVList.java | 69 +++
.../datastructure/tv/SerializableDoubleTVList.java | 69 +++
.../datastructure/tv/SerializableFloatTVList.java | 69 +++
.../datastructure/tv/SerializableIntTVList.java | 69 +++
.../datastructure/tv/SerializableLongTVList.java | 69 +++
.../udf/datastructure/tv/SerializableTVList.java | 99 ++++
.../udf/service/TemporaryQueryDataFileService.java | 136 +++++
.../iotdb/db/query/udf/service/UDFClassLoader.java | 64 +++
.../iotdb/db/query/udf/service/UDFLogWriter.java | 69 +++
.../udf/service/UDFRegistrationInformation.java} | 93 ++--
.../query/udf/service/UDFRegistrationService.java | 329 ++++++++++++
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 +
.../org/apache/iotdb/db/service/ServiceType.java | 4 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 102 ++--
.../db/sync/receiver/transfer/SyncServiceImpl.java | 20 +-
.../iotdb/db/sync/sender/transfer/SyncClient.java | 11 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 25 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 9 -
.../apache/iotdb/db/auth/AuthorityCheckerTest.java | 3 -
.../iotdb/db/engine/storagegroup/TTLTest.java | 2 +-
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 6 +-
.../iotdb/db/integration/IoTDBUDFManagementIT.java | 308 +++++++++++
.../db/integration/IoTDBUDFWindowQueryIT.java | 513 ++++++++++++++++++
.../integration/IoTDBUDTFAlignByTimeQueryIT.java | 525 +++++++++++++++++++
.../db/integration/IoTDBUDTFHybridQueryIT.java | 166 ++++++
.../db/integration/IoTDBUDTFNonAlignQueryIT.java | 337 ++++++++++++
.../apache/iotdb/db/metadata/PartialPathTest.java | 34 +-
.../iotdb/db/qp/plan/ConcatOptimizerTest.java | 7 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 213 +++++++-
.../query/dataset/UDTFAlignByTimeDataSetTest.java | 578 +++++++++++++++++++++
.../ElasticSerializableRowRecordListTest.java | 224 ++++++++
.../ElasticSerializableTVListTest.java | 251 +++++++++
.../SerializableBinaryTVListTest.java | 80 +++
.../SerializableBooleanTVListTest.java | 80 +++
.../SerializableDoubleTVListTest.java | 79 +++
.../datastructure/SerializableFloatTVListTest.java | 78 +++
.../datastructure/SerializableIntTVListTest.java | 78 +++
.../udf/datastructure/SerializableListTest.java} | 81 ++-
.../datastructure/SerializableLongTVListTest.java | 78 +++
.../SerializableRowRecordListTest.java | 143 +++++
.../udf/datastructure/SerializableTVListTest.java} | 79 ++-
.../iotdb/db/query/udf/example/Accumulator.java | 82 +++
.../apache/iotdb/db/query/udf/example/Adder.java | 78 +++
.../apache/iotdb/db/query/udf/example/Counter.java | 76 +++
.../iotdb/db/query/udf/example/Multiplier.java | 54 ++
.../SlidingSizeWindowConstructorTester0.java | 53 ++
.../SlidingSizeWindowConstructorTester1.java | 54 ++
.../SlidingTimeWindowConstructionTester.java | 61 +++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +
.../apache/iotdb/db/writelog/PerformanceTest.java | 15 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 -
.../java/org/apache/iotdb/session/SessionUT.java | 2 +-
site/src/main/.vuepress/config.js | 26 +-
.../org/apache/iotdb/tsfile/read/common/Field.java | 4 -
.../apache/iotdb/tsfile/read/common/RowRecord.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 4 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/EmptyDataSet.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 4 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 11 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +-
177 files changed, 12921 insertions(+), 1529 deletions(-)
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
index cc19a52,3e066d3..05745fd
--- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
@@@ -56,11 -53,11 +56,11 @@@ public class ClusterConfig
private boolean useAsyncApplier = true;
- private int connectionTimeoutInMS = 20 * 1000;
+ private int connectionTimeoutInMS = 20_1000;
- private int readOperationTimeoutMS = 30_1000;
+ private int readOperationTimeoutMS = 30_000;
- private int writeOperationTimeoutMS = 30_1000;
+ private int writeOperationTimeoutMS = 30_000;
private int catchUpTimeoutMS = 60_000;
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/service/BaseSyncService.java
index bffc345,e1e118f..f9bda1f
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/service/BaseSyncService.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/service/BaseSyncService.java
@@@ -102,9 -102,8 +102,8 @@@ public abstract class BaseSyncService i
if (client == null) {
throw new TException(new LeaderUnknownException(member.getAllNodes()));
}
- long commitIndex1 = 0;
try {
- commitIndex1 = client.requestCommitIndex(header, raftId);
- commitIndex = client.requestCommitIndex(header);
++ commitIndex = client.requestCommitIndex(header, raftId);
} catch (TException e) {
client.getInputProtocol().getTransport().close();
throw e;