You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/06/10 09:04:45 UTC
[incubator-iotdb] branch time_statstic updated: remove some
statistic item
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch time_statstic
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/time_statstic by this push:
new fc5daff remove some statistic item
fc5daff is described below
commit fc5daff1e18e472956df5a2dc724fbfc4764f095
Author: suyue <23...@qq.com>
AuthorDate: Mon Jun 10 17:04:21 2019 +0800
remove some statistic item
---
.../iotdb/db/cost/statistic/Measurement.java | 26 ++++-------
.../iotdb/db/cost/statistic/MeasurementMBean.java | 2 -
.../apache/iotdb/db/cost/statistic/Operation.java | 17 ++-----
.../iotdb/db/engine/filenode/FileNodeManager.java | 13 ------
.../org/apache/iotdb/db/qp/QueryProcessor.java | 6 ---
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 4 --
.../org/apache/iotdb/db/service/TSServiceImpl.java | 21 ++++-----
.../db/cost/statistic/PerformanceStatTest.java | 52 ++++++++++++++++++++++
8 files changed, 72 insertions(+), 69 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
index 650bb3e..067df38 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
@@ -111,7 +111,6 @@ public class Measurement implements MeasurementMBean, IService {
private long displayIntervalInMs;
private static final Logger LOGGER = LoggerFactory.getLogger(Measurement.class);
- private final int MS_TO_NANO = 1000_000;
private final String mbeanName = String
.format("%s:%s=%s", "org.apache.iotdb.db.cost.statistic", IoTDBConstant.JMX_TYPE,
getID().getJmxName());
@@ -172,7 +171,7 @@ public class Measurement implements MeasurementMBean, IService {
}
@Override
- public void startOneTimeStatistics(){
+ public void startOneTimeStatistics() {
stateChangeLock.lock();
try {
if (isEnableStat) {
@@ -181,10 +180,10 @@ public class Measurement implements MeasurementMBean, IService {
isEnableStat = true;
futureList.clear();
futureList.add(service.schedule(new QueueConsumerThread(), 10, TimeUnit.MILLISECONDS));
- Future future = service.schedule(()->{
+ Future future = service.schedule(() -> {
showMeasurements();
- stopStatistic();
- }, displayIntervalInMs, TimeUnit.MILLISECONDS);
+ stopStatistic();
+ }, displayIntervalInMs, TimeUnit.MILLISECONDS);
futureList.add(future);
} catch (Exception e) {
LOGGER.error("Find error when start performance statistic thread, because {}", e);
@@ -197,7 +196,7 @@ public class Measurement implements MeasurementMBean, IService {
public void stopStatistic() {
stateChangeLock.lock();
try {
- if(isEnableStat == false){
+ if (isEnableStat == false) {
return;
}
isEnableStat = false;
@@ -212,7 +211,6 @@ public class Measurement implements MeasurementMBean, IService {
} finally {
stateChangeLock.unlock();
}
-
}
/**
@@ -269,11 +267,6 @@ public class Measurement implements MeasurementMBean, IService {
return isEnableStat;
}
- @Override
- public void setEnableStat(boolean enableStat) {
- isEnableStat = enableStat;
- }
-
public long getDisplayIntervalInMs() {
return displayIntervalInMs;
}
@@ -300,8 +293,7 @@ public class Measurement implements MeasurementMBean, IService {
LOGGER.info(head);
for (Operation operation : Operation.values()) {
long cnt = operationCnt[operation.ordinal()];
- long totalInMs = 0;
- totalInMs = operationLatencies[operation.ordinal()] / 1000000;
+ long totalInMs = operationLatencies[operation.ordinal()];
String avg = String.format("%.4f", (totalInMs / (cnt + 1e-9)));
String item = String
.format("%-45s%-25s%-25s%-25s", operation.name, cnt + "", totalInMs + "", avg);
@@ -344,7 +336,7 @@ public class Measurement implements MeasurementMBean, IService {
consumer();
}
- private void consumer(){
+ private void consumer() {
int cnt = 0;
boolean allEmpty = false;
while (true) {
@@ -374,9 +366,9 @@ public class Measurement implements MeasurementMBean, IService {
}
}
}
+
private int calIndex(long x) {
- x /= MS_TO_NANO;
- for (int i = 0; i < MS_TO_NANO; i++) {
+ for (int i = 0; i < BUCKET_SIZE; i++) {
if (BUCKET_IN_MS[i] >= x) {
return i;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/MeasurementMBean.java b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/MeasurementMBean.java
index 4250f9c..5d4de7e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/MeasurementMBean.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/MeasurementMBean.java
@@ -38,8 +38,6 @@ public interface MeasurementMBean {
boolean isEnableStat();
- void setEnableStat(boolean enableStat);
-
long getDisplayIntervalInMs();
void setDisplayIntervalInMs(long displayIntervalInMs);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java
index 4e5c363..6f862ba 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java
@@ -19,20 +19,9 @@
package org.apache.iotdb.db.cost.statistic;
public enum Operation {
- EXECUTE_BATCH_SQL("EXECUTE_BATCH_SQL"),
- PARSE_SQL_TO_PHYSICAL_PLAN("1 PARSE_SQL_TO_PHYSICAL_PLAN"),
- GENERATE_AST_NODE("1.1 GENERATE_AST_NODE"),
- GENERATE_PHYSICAL_PLAN("1.2 GENERATE_PHYSICAL_PLAN"),
- EXECUTE_PHYSICAL_PLAN("2 EXECUTE_PHYSICAL_PLAN"),
- CHECK_AUTHORIZATION("2.1 CHECK_AUTHORIZATION"),
- EXECUTE_NON_QUERY("2.2 EXECUTE_NON_QUERY"),
- CONSTRUCT_TSRECORD("2.2.1 CONSTRUCT_TSRECORD"),
- GET_FILENODE_PROCESSOR("2.2.2 GET_FILENODE_PROCESSOR(ADD LOCK)"),
- INSERT_BUFFER_WRITE_OR_OVERFLOW("2.2.3 INSERT_BUFFER_WRITE_OR_OVERFLOW"),
- GET_BUFFER_WRITE_PROFESSOR("2.2.3.1 GET_BUFFER_WRITE_PROFESSOR"),
- WRITE_WAL("2.2.3.2 WRITE_WAL"),
- WRITE_MEM_TABLE("2.2.3.3 WRITE_MEM_TABLE"),
- CONSTRUCT_JDBC_RESULT("2.3 CONSTRUCT_JDBC_RESULT"),;
+ EXECUTE_BATCH("EXECUTE_BATCH"),
+ EXECUTE_ONE_SQL_IN_BATCH("EXECUTE_ONE_SQL_IN_BATCH"),
+ EXECUTE_QUERY("EXECUTE_QUERY");
public String getName() {
return name;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index 49f8f2c..4c50f9e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
@@ -36,8 +36,6 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.cost.statistic.Measurement;
-import org.apache.iotdb.db.cost.statistic.Operation;
import org.apache.iotdb.db.engine.Processor;
import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
@@ -284,15 +282,11 @@ public class FileNodeManager implements IStatistic, IService {
checkTimestamp(tsRecord);
updateStat(isMonitor, tsRecord);
- long t0 = System.nanoTime();
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- Measurement.INSTANCE.addOperationLatency(Operation.GET_FILENODE_PROCESSOR, t0);
int insertType;
try {
-
- long t2 = System.nanoTime();
long lastUpdateTime = fileNodeProcessor.getFlushLastUpdateTime(deviceId);
if (timestamp < lastUpdateTime) {
insertOverflow(fileNodeProcessor, timestamp, tsRecord, isMonitor, deviceId);
@@ -301,7 +295,6 @@ public class FileNodeManager implements IStatistic, IService {
insertBufferWrite(fileNodeProcessor, timestamp, isMonitor, tsRecord, deviceId);
insertType = 2;
}
- Measurement.INSTANCE.addOperationLatency(Operation.INSERT_BUFFER_WRITE_OR_OVERFLOW, t2);
} catch (FileNodeProcessorException e) {
LOGGER.error(String.format("Encounter an error when closing the buffer write processor %s.",
@@ -403,9 +396,7 @@ public class FileNodeManager implements IStatistic, IService {
BufferWriteProcessor bufferWriteProcessor;
String filenodeName = fileNodeProcessor.getProcessorName();
try {
- long t0 = System.nanoTime();
bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName, timestamp);
- Measurement.INSTANCE.addOperationLatency(Operation.GET_BUFFER_WRITE_PROFESSOR,t0);
} catch (FileNodeProcessorException e) {
LOGGER.error("Get the bufferwrite processor failed, the filenode is {}, insert time is {}",
filenodeName, timestamp);
@@ -430,13 +421,10 @@ public class FileNodeManager implements IStatistic, IService {
}
// write wal
- long t2 = System.nanoTime();
writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
- Measurement.INSTANCE.addOperationLatency(Operation.WRITE_WAL,t2);
// Write data
- long t4 = System.nanoTime();
long prevStartTime = fileNodeProcessor.getIntervalFileNodeStartTime(deviceId);
long prevUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
@@ -454,7 +442,6 @@ public class FileNodeManager implements IStatistic, IService {
}
throw new FileNodeManagerException(e);
}
- Measurement.INSTANCE.addOperationLatency(Operation.WRITE_MEM_TABLE,t4);
if (bufferWriteProcessor
.getFileSize() > IoTDBDescriptor.getInstance()
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java
index 356fe74..30b911e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/QueryProcessor.java
@@ -21,8 +21,6 @@ package org.apache.iotdb.db.qp;
import java.time.ZoneId;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.cost.statistic.Measurement;
-import org.apache.iotdb.db.cost.statistic.Operation;
import org.apache.iotdb.db.exception.ArgsErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.qp.IllegalASTFormatException;
@@ -69,15 +67,11 @@ public class QueryProcessor {
public PhysicalPlan parseSQLToPhysicalPlan(String sqlStr, ZoneId zoneId)
throws QueryProcessorException, ArgsErrorException, ProcessorException {
- long t0 = System.nanoTime();
AstNode astNode = parseSQLToAST(sqlStr);
- Measurement.INSTANCE.addOperationLatency(Operation.GENERATE_AST_NODE, t0);
- long t1 = System.nanoTime();
Operator operator = parseASTToOperator(astNode, zoneId);
operator = logicalOptimize(operator, executor);
PhysicalGenerator physicalGenerator = new PhysicalGenerator(executor);
PhysicalPlan qp = physicalGenerator.transformToPhysicalPlan(operator);
- Measurement.INSTANCE.addOperationLatency(Operation.GENERATE_PHYSICAL_PLAN, t1);
return qp;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
index 475fa3d..9dda4be 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
@@ -31,8 +31,6 @@ import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.PrivilegeType;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
-import org.apache.iotdb.db.cost.statistic.Measurement;
-import org.apache.iotdb.db.cost.statistic.Operation;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.ArgsErrorException;
import org.apache.iotdb.db.exception.FileNodeManagerException;
@@ -270,7 +268,6 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
String[] insertValues)
throws ProcessorException {
try {
- long t0 = System.nanoTime();
TSRecord tsRecord = new TSRecord(insertTime, deviceId);
MNode node = mManager.getNodeByDeviceIdFromCache(deviceId);
@@ -293,7 +290,6 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementList[i], value);
tsRecord.addTuple(dataPoint);
}
- Measurement.INSTANCE.addOperationLatency(Operation.CONSTRUCT_TSRECORD, t0);
return fileNodeManager.insert(tsRecord, false);
} catch (PathErrorException | FileNodeManagerException e) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 5c12d59..061305f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -436,7 +436,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
@Override
public TSExecuteBatchStatementResp executeBatchStatement(TSExecuteBatchStatementReq req)
throws TException {
- long st = System.nanoTime();
+ long t1 = System.currentTimeMillis();
try {
if (!checkLogin()) {
LOGGER.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME);
@@ -449,19 +449,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
for (String statement : statements) {
try {
- long t0 = System.nanoTime();
+ long t2 = System.currentTimeMillis();
PhysicalPlan physicalPlan = processor.parseSQLToPhysicalPlan(statement, zoneIds.get());
- Measurement.INSTANCE.addOperationLatency(Operation.PARSE_SQL_TO_PHYSICAL_PLAN, t0);
physicalPlan.setProposer(username.get());
if (physicalPlan.isQuery()) {
return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS,
"statement is query :" + statement, result);
}
-
- long t1 = System.nanoTime();
TSExecuteStatementResp resp = executeUpdateStatement(physicalPlan);
- Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_PHYSICAL_PLAN, t1);
-
if (resp.getStatus().getStatusCode().equals(TS_StatusCode.SUCCESS_STATUS)) {
result.add(Statement.SUCCESS_NO_INFO);
} else {
@@ -469,6 +464,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
isAllSuccessful = false;
batchErrorMessage = resp.getStatus().getErrorMessage();
}
+ Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_ONE_SQL_IN_BATCH, t2);
} catch (Exception e) {
String errMessage = String.format(
"Fail to generate physcial plan and execute for statement "
@@ -492,7 +488,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage(), null);
}
finally {
- Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_BATCH_SQL, st);
+ Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_BATCH, t1);
}
}
@@ -565,7 +561,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
@Override
public TSExecuteStatementResp executeQueryStatement(TSExecuteStatementReq req) throws TException {
-
+ long t1 = System.currentTimeMillis();
try {
if (!checkLogin()) {
LOGGER.info("{}: Not login.", IoTDBConstant.GLOBAL_DB_NAME);
@@ -666,6 +662,9 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
LOGGER.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e);
return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage());
}
+ finally {
+ Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_QUERY, t1);
+ }
}
protected void checkFileLevelSet(List<Path> paths) throws PathErrorException {
@@ -767,15 +766,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
// Do we need to add extra information of executive condition
boolean execRet;
try {
- long t1 = System.nanoTime();
execRet = executeNonQuery(plan);
- Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_NON_QUERY, t1);
} catch (ProcessorException e) {
LOGGER.debug("meet error while processing non-query. ", e);
return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage());
}
- long t2 = System.nanoTime();
TS_StatusCode statusCode = execRet ? TS_StatusCode.SUCCESS_STATUS : TS_StatusCode.ERROR_STATUS;
String msg = execRet ? "Execute successfully" : "Execute statement error.";
TSExecuteStatementResp resp = getTSExecuteStatementResp(statusCode, msg);
@@ -785,7 +781,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
TSOperationHandle operationHandle;
operationHandle = new TSOperationHandle(operationId, false);
resp.setOperationHandle(operationHandle);
- Measurement.INSTANCE.addOperationLatency(Operation.CONSTRUCT_JDBC_RESULT, t2);
return resp;
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java b/iotdb/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java
index 8f6c325..15e5167 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java
@@ -1,5 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.iotdb.db.cost.statistic;
+import org.junit.Test;
+
public class PerformanceStatTest {
+ @Test
+ public void test() {
+ Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_BATCH, System.currentTimeMillis());
+ Measurement.INSTANCE
+ .addOperationLatency(Operation.EXECUTE_BATCH, System.currentTimeMillis() - 8000000);
+
+ try {
+ Measurement.INSTANCE.start();
+ Measurement.INSTANCE.startContinuousStatistics();
+ Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_BATCH, System.currentTimeMillis());
+ Measurement.INSTANCE
+ .addOperationLatency(Operation.EXECUTE_BATCH, System.currentTimeMillis() - 8000000);
+ Thread.currentThread().sleep(2000);
+ Measurement.INSTANCE.stopStatistic();
+ Measurement.INSTANCE.stopStatistic();
+ Measurement.INSTANCE.stopStatistic();
+ System.out.println("After stopStatistic!");
+ Thread.currentThread().sleep(1000);
+ Measurement.INSTANCE.startContinuousStatistics();
+ System.out.println("ReStart!");
+ Thread.currentThread().sleep(2000);
+ Measurement.INSTANCE.startContinuousStatistics();
+ System.out.println("ReStart2!");
+ Thread.currentThread().sleep(2000);
+ Measurement.INSTANCE.stopStatistic();
+ System.out.println("After stopStatistic2!");
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ Measurement.INSTANCE.stop();
+ }
+ }
}