You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2020/12/04 01:56:53 UTC
[iotdb] branch virtual_partition updated (1460c1a -> d58fb2e)
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a change to branch virtual_partition
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
from 1460c1a change partition
add 8657b13 Add Apache Pulsar to IoTDB example (#2122)
add d242932 Update compaction level list delete (#2152)
add 18bb46a [IOTDB-890] SDT implementation (#2087)
add 467897e add charset UTF-8 (#2155)
add f6cd1f9 [IOTDB-1035] Fix bug in getDeviceTimeseriesMetadata when querying non-exist device (#2150)
add 8e8384f Fix File Not Found when serializing TsFileResources (#2161)
add e95cebc Fix flink-iotdb set storage group bug (#2165)
add ccd6fab [IOTDB-1032]Refactor site/pom.xml and move v0.11 to 0.12 in all pom files (#2139)
add df22109 [IOTDB-1011] Memtable sort in query (#2144)
add 7dbc215 doc: fix dead links in the README (#2174)
add d951e82 [Mem control] Move one of the insert check out from the sg lock (#2154)
add 8aa9fef Remove duplicated static resps(column header) (#2164)
add bf0fb06 fix insert partial tablet with binary NullPointer bug (#2180)
add 20fed99 [IOTDB-1035] Add test for ReadChunkMetadataInDevice (#2157)
new d58fb2e Merge branch 'master' of https://github.com/apache/iotdb into virtual_partition
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
README_ZH.md | 2 +-
antlr/pom.xml | 2 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 5 -
cli/pom.xml | 4 +-
client-cpp/client-cpp-example/pom.xml | 4 +-
client-cpp/pom.xml | 4 +-
client-py/pom.xml | 2 +-
client-py/pypi/setup.py | 2 +-
compile-tools/boost/pom.xml | 2 +-
compile-tools/pom.xml | 4 +-
compile-tools/thrift/pom.xml | 2 +-
cross-tests/pom.xml | 2 +-
distribution/pom.xml | 2 +-
docker/src/main/Dockerfile | 4 +-
docs/Download/README.md | 47 +-
docs/SystemDesign/TsFile/Format.md | 2 +-
docs/UserGuide/Client/Command Line Interface.md | 1 +
docs/UserGuide/Concept/SDT.md | 45 ++
docs/UserGuide/Operation Manual/SQL Reference.md | 10 +-
docs/zh/Download/README.md | 47 +-
docs/zh/SystemDesign/TsFile/Format.md | 2 +-
docs/zh/UserGuide/Client/Command Line Interface.md | 2 +-
.../zh/UserGuide/Concept/SDT.md | 24 +-
.../UserGuide/Ecosystem Integration/Hive TsFile.md | 6 +-
.../zh/UserGuide/Operation Manual/SQL Reference.md | 10 +-
docs/zh/UserGuide/Server/Download.md | 14 +-
example/flink/pom.xml | 2 +-
example/hadoop/pom.xml | 6 +-
example/jdbc/pom.xml | 2 +-
example/kafka/pom.xml | 2 +-
.../main/java/org/apache/iotdb/kafka/Constant.java | 2 +
example/mqtt/pom.xml | 2 +-
example/pom.xml | 3 +-
example/{tsfile => pulsar}/pom.xml | 18 +-
.../java/org/apache/iotdb/pulsar}/Constant.java | 18 +-
.../org/apache/iotdb/pulsar/PulsarConsumer.java | 108 ++++
.../apache/iotdb/pulsar/PulsarConsumerThread.java | 70 +++
.../org/apache/iotdb/pulsar/PulsarProducer.java | 82 +++
example/rocketmq/pom.xml | 2 +-
example/session/pom.xml | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 14 +
example/tsfile/pom.xml | 2 +-
flink-iotdb-connector/pom.xml | 2 +-
.../java/org/apache/iotdb/flink/IoTDBSink.java | 25 +-
flink-tsfile-connector/pom.xml | 2 +-
grafana/pom.xml | 2 +-
hadoop/pom.xml | 4 +-
hive-connector/pom.xml | 6 +-
jdbc/pom.xml | 2 +-
pom.xml | 2 +-
server/pom.xml | 2 +-
.../resources/conf/iotdb-engine.properties | 10 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 19 +
.../level/LevelCompactionTsFileManagement.java | 41 +-
.../engine/compaction/utils/CompactionUtils.java | 2 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 2 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 52 +-
.../db/engine/memtable/IWritableMemChunk.java | 24 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 35 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 21 +-
.../engine/storagegroup/StorageGroupProcessor.java | 12 +
.../db/engine/storagegroup/TsFileProcessor.java | 53 +-
.../db/engine/storagegroup/TsFileResource.java | 2 +-
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 6 +-
.../org/apache/iotdb/db/service/StaticResps.java | 98 +---
.../org/apache/iotdb/db/service/TSServiceImpl.java | 108 +---
.../iotdb/db/utils/datastructure/TVList.java | 128 ++---
.../db/engine/memtable/PrimitiveMemTableTest.java | 2 +-
.../db/integration/IoTDBInsertWithQueryIT.java | 503 ++++++++++++++++++
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 379 +++++++++++++-
service-rpc/pom.xml | 2 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 7 +-
session/pom.xml | 2 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 47 ++
site/README-zh.md | 8 +-
site/README.md | 15 +-
site/pom.xml | 568 +++++++++++++++++----
site/src/main/.vuepress/config.js | 198 ++++++-
site/src/main/package.json | 6 +-
spark-iotdb-connector/pom.xml | 2 +-
spark-tsfile/pom.xml | 4 +-
thrift/pom.xml | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/encoding/encoder/SDTEncoder.java | 448 ++++++++++++++++
.../iotdb/tsfile/read/TsFileSequenceReader.java | 18 +-
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 121 ++++-
.../apache/iotdb/tsfile/write/record/Tablet.java | 5 +-
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 176 +++++++
.../tsfile/read/TsFileSequenceReaderTest.java | 41 +-
92 files changed, 3171 insertions(+), 653 deletions(-)
create mode 100644 docs/UserGuide/Concept/SDT.md
copy server/file-changelists/role-changelist.md => docs/zh/UserGuide/Concept/SDT.md (53%)
copy example/{tsfile => pulsar}/pom.xml (79%)
copy example/{tsfile/src/main/java/org/apache/iotdb/tsfile => pulsar/src/main/java/org/apache/iotdb/pulsar}/Constant.java (66%)
create mode 100644 example/pulsar/src/main/java/org/apache/iotdb/pulsar/PulsarConsumer.java
create mode 100644 example/pulsar/src/main/java/org/apache/iotdb/pulsar/PulsarConsumerThread.java
create mode 100644 example/pulsar/src/main/java/org/apache/iotdb/pulsar/PulsarProducer.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertWithQueryIT.java
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/SDTEncoder.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SDTEncoderTest.java
[iotdb] 01/01: Merge branch 'master' of
https://github.com/apache/iotdb into virtual_partition
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch virtual_partition
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d58fb2e8ad1fd7034ba98930225b60cec6b838e1
Merge: 1460c1a 20fed99
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Fri Dec 4 09:56:32 2020 +0800
Merge branch 'master' of https://github.com/apache/iotdb into virtual_partition
# Conflicts:
# server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
README.md | 2 +-
README_ZH.md | 2 +-
antlr/pom.xml | 2 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 5 -
cli/pom.xml | 4 +-
client-cpp/client-cpp-example/pom.xml | 4 +-
client-cpp/pom.xml | 4 +-
client-py/pom.xml | 2 +-
client-py/pypi/setup.py | 2 +-
compile-tools/boost/pom.xml | 2 +-
compile-tools/pom.xml | 4 +-
compile-tools/thrift/pom.xml | 2 +-
cross-tests/pom.xml | 2 +-
distribution/pom.xml | 2 +-
docker/src/main/Dockerfile | 4 +-
docs/Download/README.md | 47 +-
docs/SystemDesign/TsFile/Format.md | 2 +-
docs/UserGuide/Client/Command Line Interface.md | 1 +
docs/UserGuide/Concept/SDT.md | 45 ++
docs/UserGuide/Operation Manual/SQL Reference.md | 10 +-
docs/zh/Download/README.md | 47 +-
docs/zh/SystemDesign/TsFile/Format.md | 2 +-
docs/zh/UserGuide/Client/Command Line Interface.md | 2 +-
.../pom.xml => docs/zh/UserGuide/Concept/SDT.md | 45 +-
.../UserGuide/Ecosystem Integration/Hive TsFile.md | 6 +-
.../zh/UserGuide/Operation Manual/SQL Reference.md | 10 +-
docs/zh/UserGuide/Server/Download.md | 14 +-
example/flink/pom.xml | 2 +-
example/hadoop/pom.xml | 6 +-
example/jdbc/pom.xml | 2 +-
example/kafka/pom.xml | 2 +-
.../main/java/org/apache/iotdb/kafka/Constant.java | 2 +
example/mqtt/pom.xml | 2 +-
example/pom.xml | 3 +-
example/{jdbc => pulsar}/pom.xml | 18 +-
.../java/org/apache/iotdb/pulsar/Constant.java | 30 ++
.../org/apache/iotdb/pulsar/PulsarConsumer.java | 108 ++++
.../apache/iotdb/pulsar/PulsarConsumerThread.java | 70 +++
.../org/apache/iotdb/pulsar/PulsarProducer.java | 82 +++
example/rocketmq/pom.xml | 2 +-
example/session/pom.xml | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 14 +
example/tsfile/pom.xml | 2 +-
flink-iotdb-connector/pom.xml | 2 +-
.../java/org/apache/iotdb/flink/IoTDBSink.java | 25 +-
flink-tsfile-connector/pom.xml | 2 +-
grafana/pom.xml | 2 +-
hadoop/pom.xml | 4 +-
hive-connector/pom.xml | 6 +-
jdbc/pom.xml | 2 +-
pom.xml | 2 +-
server/pom.xml | 2 +-
.../resources/conf/iotdb-engine.properties | 10 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 19 +
.../level/LevelCompactionTsFileManagement.java | 41 +-
.../engine/compaction/utils/CompactionUtils.java | 2 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 2 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 52 +-
.../db/engine/memtable/IWritableMemChunk.java | 24 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 35 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 21 +-
.../engine/storagegroup/StorageGroupProcessor.java | 12 +
.../db/engine/storagegroup/TsFileProcessor.java | 53 +-
.../db/engine/storagegroup/TsFileResource.java | 2 +-
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 6 +-
.../org/apache/iotdb/db/service/StaticResps.java | 98 +---
.../org/apache/iotdb/db/service/TSServiceImpl.java | 108 +---
.../iotdb/db/utils/datastructure/TVList.java | 128 ++---
.../db/engine/memtable/PrimitiveMemTableTest.java | 2 +-
.../db/integration/IoTDBInsertWithQueryIT.java | 503 ++++++++++++++++++
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 379 +++++++++++++-
service-rpc/pom.xml | 2 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 7 +-
session/pom.xml | 2 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 47 ++
site/README-zh.md | 8 +-
site/README.md | 15 +-
site/pom.xml | 568 +++++++++++++++++----
site/src/main/.vuepress/config.js | 198 ++++++-
site/src/main/package.json | 6 +-
spark-iotdb-connector/pom.xml | 2 +-
spark-tsfile/pom.xml | 4 +-
thrift/pom.xml | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/encoding/encoder/SDTEncoder.java | 448 ++++++++++++++++
.../iotdb/tsfile/read/TsFileSequenceReader.java | 18 +-
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 121 ++++-
.../apache/iotdb/tsfile/write/record/Tablet.java | 5 +-
.../iotdb/tsfile/encoding/SDTEncoderTest.java | 176 +++++++
.../tsfile/read/TsFileSequenceReaderTest.java | 41 +-
92 files changed, 3200 insertions(+), 657 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index db2e453,5540fbb..0502b5f
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@@ -925,12 -840,21 +926,30 @@@ public class StorageEngine implements I
list.forEach(storageGroupProcessor -> storageGroupProcessor.getTsFileManagement().readUnLock());
}
+ static class InstanceHolder {
+
+ private static final StorageEngine INSTANCE = new StorageEngine();
+
+ private InstanceHolder() {
+ // forbidding instantiation
+ }
+ }
++
+ /**
+ * block insertion if the insertion is rejected by memory control
+ */
+ public static void blockInsertionIfReject() throws WriteProcessException {
+ long startTime = System.currentTimeMillis();
+ while (SystemInfo.getInstance().isRejected()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked());
+ if (System.currentTimeMillis() - startTime > config.getMaxWaitingTimeWhenInsertBlocked()) {
+ throw new WriteProcessException("System rejected over " + config.getMaxWaitingTimeWhenInsertBlocked() +
+ "ms");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
}