You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/06/14 02:17:31 UTC
[incubator-iotdb] branch time_statstic updated (67057aa -> 1867e72)
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a change to branch time_statstic
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 67057aa fix bug
add 5cc46a9 [IOTDB-96]Refactor author query (#179)
add 343c7bd [IOTDB-108]Fix mistakes in doc (#187)
add e2fb430 fix that ignoreTimeStamp in response is incorrectly set and used (#191)
add 5c15ac3 Organize properties (#189)
add 68844b2 fix missing wal log node after recovery
add 6f40c05 Merge pull request #186 from apache/fix_missing_wal_node
add a7c0221 simplify Path construction
add 8a642a4 Merge pull request #193 from apache/f_construct_path
add 4d84ef9 Fix a doc bug of QuickStart.md
add 36484ac Merge pull request #194 from sontallive/patch-3
new 1867e72 merge master for create pr
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/Documentation/QuickStart.md | 2 +-
.../UserGuideV0.7.0/5-SQL Documentation.md | 13 +-
.../apache/iotdb/cli/client/AbstractClient.java | 4 +
iotdb/iotdb/conf/iotdb-engine.properties | 68 ++--
iotdb/iotdb/conf/iotdb-sync-client.properties | 4 +-
.../apache/iotdb/db/auth/entity/PathPrivilege.java | 1 -
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +-
.../engine/bufferwrite/BufferWriteProcessor.java | 24 +-
.../iotdb/db/engine/filenode/FileNodeManager.java | 14 +-
.../db/engine/overflow/io/OverflowProcessor.java | 18 +-
.../db/qp/executor/IQueryProcessExecutor.java | 3 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 263 ++++++++-----
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 17 +-
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 4 +
.../iotdb/db/qp/physical/crud/InsertPlan.java | 2 +-
.../iotdb/db/qp/physical/sys/AuthorPlan.java | 6 +
.../dataset/AuthDataSet.java} | 37 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 195 ++++++----
.../org/apache/iotdb/db/sync/conf/Constans.java | 10 +
.../iotdb/db/sync/conf/SyncSenderConfig.java | 24 +-
.../iotdb/db/sync/conf/SyncSenderDescriptor.java | 7 +-
.../apache/iotdb/db/sync/sender/SyncSender.java | 10 +
.../iotdb/db/sync/sender/SyncSenderImpl.java | 106 +++---
.../iotdb/db/integration/IoTDBAuthorizationIT.java | 416 +++++++++++----------
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 6 +
.../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 9 +
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 9 +-
service-rpc/src/main/thrift/rpc.thrift | 1 +
.../org/apache/iotdb/tsfile/read/common/Path.java | 11 +-
30 files changed, 776 insertions(+), 522 deletions(-)
copy iotdb/src/main/java/org/apache/iotdb/db/{qp/logical/crud/FromOperator.java => query/dataset/AuthDataSet.java} (55%)
[incubator-iotdb] 01/01: merge master for create pr
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch time_statstic
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 1867e72814a3f9f4e7d27e85e80aa27e761f95b7
Merge: 67057aa 36484ac
Author: suyue <23...@qq.com>
AuthorDate: Fri Jun 14 10:17:18 2019 +0800
merge master for create pr
docs/Documentation/QuickStart.md | 2 +-
.../UserGuideV0.7.0/5-SQL Documentation.md | 13 +-
.../apache/iotdb/cli/client/AbstractClient.java | 4 +
iotdb/iotdb/conf/iotdb-engine.properties | 68 ++--
iotdb/iotdb/conf/iotdb-sync-client.properties | 4 +-
.../apache/iotdb/db/auth/entity/PathPrivilege.java | 1 -
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +-
.../engine/bufferwrite/BufferWriteProcessor.java | 24 +-
.../iotdb/db/engine/filenode/FileNodeManager.java | 14 +-
.../db/engine/overflow/io/OverflowProcessor.java | 18 +-
.../db/qp/executor/IQueryProcessExecutor.java | 3 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 263 ++++++++-----
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 17 +-
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 4 +
.../iotdb/db/qp/physical/crud/InsertPlan.java | 2 +-
.../iotdb/db/qp/physical/sys/AuthorPlan.java | 6 +
.../apache/iotdb/db/query/dataset/AuthDataSet.java | 53 +++
.../org/apache/iotdb/db/service/TSServiceImpl.java | 195 ++++++----
.../org/apache/iotdb/db/sync/conf/Constans.java | 10 +
.../iotdb/db/sync/conf/SyncSenderConfig.java | 24 +-
.../iotdb/db/sync/conf/SyncSenderDescriptor.java | 7 +-
.../apache/iotdb/db/sync/sender/SyncSender.java | 10 +
.../iotdb/db/sync/sender/SyncSenderImpl.java | 106 +++---
.../iotdb/db/integration/IoTDBAuthorizationIT.java | 416 +++++++++++----------
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 6 +
.../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 9 +
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 9 +-
service-rpc/src/main/thrift/rpc.thrift | 1 +
.../org/apache/iotdb/tsfile/read/common/Path.java | 11 +-
30 files changed, 808 insertions(+), 506 deletions(-)
diff --cc iotdb/iotdb/conf/iotdb-engine.properties
index f724051,848696c..1c220c3
--- a/iotdb/iotdb/conf/iotdb-engine.properties
+++ b/iotdb/iotdb/conf/iotdb-engine.properties
@@@ -215,15 -237,9 +237,17 @@@ sync_server_port=555
# The default is to allow all IP to sync
IP_white_list=0.0.0.0/0
- # Choose a sync strategy of loading historical data:
- #1. It's more likely to update historical data, please choose "true".
- #2. It's more likely not to update historical data or you don't know exactly, please choose "false".
+ # The processing strategy chosen by the sync server when merging the sync data.
+ # 1. If the sync data accounts for more than 50% of the update of the historical data (compared with the latest timestamp of the local storage group data),then it is recommended to select strategy 1.
+ # Setting the parameter to true, which has a greater impact on the write performance of the IoTDB system and occupies less CPU of the machine.
+ # 2. If the sync data accounts for less than 50% of the update of the historical data (compared with the latest timestamp of the local storage group data),then it is recommended to select strategy 2.
+ # Setting the parameter to false, which has little impact on the write performance of IoTDB system and takes up a large amount of CPU power.
update_historical_data_possibility=false
+
+# performance statistic configuration
+# Is stat performance of sub-module enable
+enable_performance_stat=false
+# The interval of display statistic result in ms.
+performance_stat_display_interval=60000
+# The memory used for performance_stat in kb.
+performance_stat_memory_in_kb=20
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index 4c50f9e,656ece6..2d7429a
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
@@@ -419,11 -421,12 +423,13 @@@ public class FileNodeManager implement
throw new FileNodeManagerException(e);
}
}
+
// write wal
- writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
-
-
+ try {
+ writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
+ } catch (IOException e) {
+ throw new FileNodeManagerException(e);
+ }
// Write data
long prevStartTime = fileNodeProcessor.getIntervalFileNodeStartTime(deviceId);
long prevUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
diff --cc iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
index 9dda4be,e772bf5..f1c2c86
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
@@@ -52,6 -57,8 +57,7 @@@ import org.apache.iotdb.db.qp.physical.
import org.apache.iotdb.db.qp.physical.sys.MetadataPlan;
import org.apache.iotdb.db.qp.physical.sys.PropertyPlan;
import org.apache.iotdb.db.query.context.QueryContext;
+ import org.apache.iotdb.db.query.dataset.AuthDataSet;
-import org.apache.iotdb.db.query.executor.EngineQueryRouter;
import org.apache.iotdb.db.query.fill.IFill;
import org.apache.iotdb.db.utils.AuthUtils;
import org.apache.iotdb.db.utils.LoadDataUtils;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 061305f,c502f46..4850843
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@@ -662,11 -589,118 +598,121 @@@ public class TSServiceImpl implements T
LOGGER.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e);
return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage());
}
+ finally {
+ Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_QUERY, t1);
+ }
}
+ private TSExecuteStatementResp executeAuthQuery(PhysicalPlan plan, List<String> columns) {
+ TSExecuteStatementResp resp = getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, "");
+ resp.setIgnoreTimeStamp(true);
+ AuthorPlan authorPlan = (AuthorPlan) plan;
+ switch (authorPlan.getAuthorType()) {
+ case LIST_ROLE:
+ columns.add(ROLE);
+ break;
+ case LIST_USER:
+ columns.add(USER);
+ break;
+ case LIST_ROLE_USERS:
+ columns.add(USER);
+ break;
+ case LIST_USER_ROLES:
+ columns.add(ROLE);
+ break;
+ case LIST_ROLE_PRIVILEGE:
+ columns.add(PRIVILEGE);
+ break;
+ case LIST_USER_PRIVILEGE:
+ columns.add(ROLE);
+ columns.add(PRIVILEGE);
+ break;
+ default:
+ return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, String.format("%s is not an "
+ + "auth query", authorPlan.getAuthorType()));
+ }
+ return resp;
+ }
+
+ private TSExecuteStatementResp executeDataQuery(PhysicalPlan plan, List<String> columns)
+ throws AuthException, TException {
+ List<Path> paths;
+ paths = plan.getPaths();
+
+ // check seriesPath exists
+ if (paths.isEmpty()) {
+ return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, "Timeseries does not exist.");
+ }
+
+ // check file level set
+
+ try {
+ checkFileLevelSet(paths);
+ } catch (PathErrorException e) {
+ LOGGER.error("meet error while checking file level.", e);
+ return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage());
+ }
+
+ // check permissions
+ if (!checkAuthorization(paths, plan)) {
+ return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS,
+ "No permissions for this query.");
+ }
+
+ TSExecuteStatementResp resp = getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, "");
+ // Restore column header of aggregate to func(column_name), only
+ // support single aggregate function for now
+ if (plan instanceof QueryPlan) {
+ switch (plan.getOperatorType()) {
+ case QUERY:
+ case FILL:
+ for (Path p : paths) {
+ columns.add(p.getFullPath());
+ }
+ break;
+ case AGGREGATION:
+ case GROUPBY:
+ List<String> aggregations = plan.getAggregations();
+ if (aggregations.size() != paths.size()) {
+ for (int i = 1; i < paths.size(); i++) {
+ aggregations.add(aggregations.get(0));
+ }
+ }
+ for (int i = 0; i < paths.size(); i++) {
+ columns.add(aggregations.get(i) + "(" + paths.get(i).getFullPath() + ")");
+ }
+ break;
+ default:
+ throw new TException("unsupported query type: " + plan.getOperatorType());
+ }
+ } else {
+ Operator.OperatorType type = plan.getOperatorType();
+ switch (type) {
+ case QUERY:
+ case FILL:
+ for (Path p : paths) {
+ columns.add(p.getFullPath());
+ }
+ break;
+ case AGGREGATION:
+ case GROUPBY:
+ List<String> aggregations = plan.getAggregations();
+ if (aggregations.size() != paths.size()) {
+ for (int i = 1; i < paths.size(); i++) {
+ aggregations.add(aggregations.get(0));
+ }
+ }
+ for (int i = 0; i < paths.size(); i++) {
+ columns.add(aggregations.get(i) + "(" + paths.get(i).getFullPath() + ")");
+ }
+ break;
+ default:
+ throw new TException("not support " + type + " in new read process");
+ }
+ }
+ return resp;
+ }
+
protected void checkFileLevelSet(List<Path> paths) throws PathErrorException {
MManager.getInstance().checkFileLevel(paths);
}