You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2018/11/26 13:44:49 UTC
[incubator-skywalking] branch mysql-style updated: Finish the code
base of MySQL style storage.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch mysql-style
in repository https://gitbox.apache.org/repos/asf/incubator-skywalking.git
The following commit(s) were added to refs/heads/mysql-style by this push:
new 065e122 Finish the code base of MySQL style storage.
065e122 is described below
commit 065e1220079531f4e83d6caf6606adf42c0f2882
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Mon Nov 26 21:44:39 2018 +0800
Finish the code base of MySQL style storage.
---
.../annotation/StorageAnnotationListener.java | 2 +-
.../oap/server/core/storage/model/ColumnName.java | 19 +---
.../plugin/jdbc/h2/dao/H2AlarmQueryDAO.java | 7 +-
.../plugin/jdbc/h2/dao/H2TableInstaller.java | 4 +-
.../plugin/jdbc/mysql/MySQLTableInstaller.java | 121 ++++++++++++++++++++-
5 files changed, 131 insertions(+), 22 deletions(-)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/StorageAnnotationListener.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/StorageAnnotationListener.java
index 7d9da52..9a71b61 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/StorageAnnotationListener.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/StorageAnnotationListener.java
@@ -71,7 +71,7 @@ public class StorageAnnotationListener implements AnnotationListener, IModelGett
for (Field field : fields) {
if (field.isAnnotationPresent(Column.class)) {
Column column = field.getAnnotation(Column.class);
- modelColumns.add(new ModelColumn(new ColumnName(column.columnName(), column.columnName()), field.getType(), column.matchQuery()));
+ modelColumns.add(new ModelColumn(new ColumnName(column.columnName()), field.getType(), column.matchQuery()));
if (logger.isDebugEnabled()) {
logger.debug("The field named {} with the {} type", column.columnName(), field.getType());
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
index d504867..97e0514 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
@@ -19,31 +19,22 @@
package org.apache.skywalking.oap.server.core.storage.model;
/**
+ * Short column name unsupported for now. No define in @Column annotation. The storage implementation need to use name to do match.
+ *
* @author peng-yongsheng
*/
public class ColumnName {
private String fullName;
- private String shortName;
- private boolean useShortName = false;
- public ColumnName(String fullName, String shortName) {
+ public ColumnName(String fullName) {
this.fullName = fullName;
- this.shortName = shortName;
}
public String getName() {
- return useShortName ? shortName : fullName;
- }
-
- public void useShortName() {
- this.useShortName = true;
+ return fullName;
}
public void setName(String name) {
- if (useShortName) {
- shortName = name;
- } else {
- fullName = name;
- }
+ fullName = name;
}
}
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2AlarmQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2AlarmQueryDAO.java
index 6662100..9b8a5bb 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2AlarmQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2AlarmQueryDAO.java
@@ -25,7 +25,6 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import org.apache.skywalking.oap.server.core.alarm.AlarmRecord;
-import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord;
import org.apache.skywalking.oap.server.core.query.entity.AlarmMessage;
import org.apache.skywalking.oap.server.core.query.entity.Alarms;
import org.apache.skywalking.oap.server.core.source.Scope;
@@ -49,12 +48,12 @@ public class H2AlarmQueryDAO implements IAlarmQueryDAO {
StringBuilder sql = new StringBuilder();
List<Object> parameters = new ArrayList<>(10);
- sql.append("from ").append(SegmentRecord.INDEX_NAME).append(" where ");
+ sql.append("from ").append(AlarmRecord.INDEX_NAME).append(" where ");
sql.append(" 1=1 ");
if (startTB != 0 && endTB != 0) {
- sql.append(" and ").append(SegmentRecord.TIME_BUCKET).append(" >= ?");
+ sql.append(" and ").append(AlarmRecord.TIME_BUCKET).append(" >= ?");
parameters.add(startTB);
- sql.append(" and ").append(SegmentRecord.TIME_BUCKET).append(" <= ?");
+ sql.append(" and ").append(AlarmRecord.TIME_BUCKET).append(" <= ?");
parameters.add(endTB);
}
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
index 34ef6e0..7daf12c 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
@@ -75,7 +75,7 @@ public class H2TableInstaller extends ModelInstaller {
for (int i = 0; i < model.getColumns().size(); i++) {
ModelColumn column = model.getColumns().get(i);
ColumnName name = column.getColumnName();
- tableCreateSQL.appendLine(name.getName() + " " + getColumnType(column.getType()) + (i != model.getColumns().size() - 1 ? "," : ""));
+ tableCreateSQL.appendLine(name.getName() + " " + getColumnType(model, name, column.getType()) + (i != model.getColumns().size() - 1 ? "," : ""));
}
tableCreateSQL.appendLine(")");
@@ -93,7 +93,7 @@ public class H2TableInstaller extends ModelInstaller {
}
- protected String getColumnType(Class<?> type) {
+ protected String getColumnType(Model model, ColumnName name, Class<?> type) {
if (Integer.class.equals(type) || int.class.equals(type)) {
return "INT";
} else if (Long.class.equals(type) || long.class.equals(type)) {
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
index 14bf6cb..eeced14 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
@@ -21,13 +21,20 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql;
import java.sql.Connection;
import java.sql.SQLException;
import org.apache.skywalking.oap.server.core.analysis.indicator.IntKeyLongValueArray;
+import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord;
+import org.apache.skywalking.oap.server.core.register.RegisterSource;
+import org.apache.skywalking.oap.server.core.source.Scope;
import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.model.ColumnName;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder;
import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TableInstaller;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Extend H2TableInstaller but match MySQL SQL syntax.
@@ -35,6 +42,8 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TableInstal
* @author wusheng
*/
public class MySQLTableInstaller extends H2TableInstaller {
+ private static final Logger logger = LoggerFactory.getLogger(MySQLTableInstaller.class);
+
public MySQLTableInstaller(ModuleManager moduleManager) {
super(moduleManager);
/**
@@ -44,6 +53,12 @@ public class MySQLTableInstaller extends H2TableInstaller {
this.overrideColumnName("match", "match_num");
}
+ @Override protected void createTable(Client client, Model model) throws StorageException {
+ super.createTable(client, model);
+ JDBCHikariCPClient jdbcHikariCPClient = (JDBCHikariCPClient)client;
+ this.createIndexes(jdbcHikariCPClient, model);
+ }
+
@Override protected void deleteTable(Client client, Model model) throws StorageException {
JDBCHikariCPClient jdbcClient = (JDBCHikariCPClient)client;
try (Connection connection = jdbcClient.getConnection()) {
@@ -54,7 +69,7 @@ public class MySQLTableInstaller extends H2TableInstaller {
}
@Override
- protected String getColumnType(Class<?> type) {
+ protected String getColumnType(Model model, ColumnName name, Class<?> type) {
if (Integer.class.equals(type) || int.class.equals(type)) {
return "INT";
} else if (Long.class.equals(type) || long.class.equals(type)) {
@@ -62,6 +77,10 @@ public class MySQLTableInstaller extends H2TableInstaller {
} else if (Double.class.equals(type) || double.class.equals(type)) {
return "DOUBLE";
} else if (String.class.equals(type)) {
+ if (Scope.Segment.equals(model.getSource())) {
+ if (name.getName().equals(SegmentRecord.TRACE_ID) || name.getName().equals(SegmentRecord.SEGMENT_ID))
+ return "VARCHAR(300)";
+ }
return "VARCHAR(2000)";
} else if (IntKeyLongValueArray.class.equals(type)) {
return "MEDIUMTEXT";
@@ -71,4 +90,104 @@ public class MySQLTableInstaller extends H2TableInstaller {
throw new IllegalArgumentException("Unsupported data type: " + type.getName());
}
}
+
+ protected void createIndexes(JDBCHikariCPClient client, Model model) throws StorageException {
+ switch (model.getSource()) {
+ case ServiceInventory:
+ case ServiceInstanceInventory:
+ case NetworkAddress:
+ case EndpointInventory:
+ createInventoryIndexes(client, model);
+ return;
+ case Segment:
+ createSegmentIndexes(client, model);
+ return;
+ case Alarm:
+ createAlarmIndexes(client, model);
+ return;
+ default:
+ createIndexesForAllIndicators(client, model);
+
+ }
+ }
+
+ private void createIndexesForAllIndicators(JDBCHikariCPClient client, Model model) throws StorageException {
+ try (Connection connection = client.getConnection()) {
+ SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_TIME_BUCKET ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(SegmentRecord.TIME_BUCKET).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+ } catch (JDBCClientException e) {
+ throw new StorageException(e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new StorageException(e.getMessage(), e);
+ }
+ }
+
+ private void createAlarmIndexes(JDBCHikariCPClient client, Model model) throws StorageException {
+ try (Connection connection = client.getConnection()) {
+ SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_TIME_BUCKET ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(SegmentRecord.TIME_BUCKET).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+ } catch (JDBCClientException e) {
+ throw new StorageException(e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new StorageException(e.getMessage(), e);
+ }
+ }
+
+ private void createSegmentIndexes(JDBCHikariCPClient client, Model model) throws StorageException {
+ try (Connection connection = client.getConnection()) {
+ SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_TRACE_ID ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(SegmentRecord.TRACE_ID).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+
+ tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_ENDPOINT_ID ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(SegmentRecord.ENDPOINT_ID).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+
+ tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_LATENCY ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(SegmentRecord.LATENCY).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+
+ tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_TIME_BUCKET ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(SegmentRecord.TIME_BUCKET).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+ } catch (JDBCClientException e) {
+ throw new StorageException(e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new StorageException(e.getMessage(), e);
+ }
+ }
+
+ private void createInventoryIndexes(JDBCHikariCPClient client, Model model) throws StorageException {
+ try (Connection connection = client.getConnection()) {
+ SQLBuilder tableIndexSQL = new SQLBuilder("CREATE UNIQUE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_SEQ ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(RegisterSource.SEQUENCE).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+
+ tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase()).append("_TIME ");
+ tableIndexSQL.append("ON ").append(model.getName()).append("(").append(RegisterSource.HEARTBEAT_TIME).append(", ").append(RegisterSource.REGISTER_TIME).append(")");
+ createIndex(client, connection, model, tableIndexSQL);
+ } catch (JDBCClientException e) {
+ throw new StorageException(e.getMessage(), e);
+ } catch (SQLException e) {
+ throw new StorageException(e.getMessage(), e);
+ }
+ }
+
+ private void createIndex(JDBCHikariCPClient client, Connection connection, Model model,
+ SQLBuilder indexSQL) throws JDBCClientException {
+ if (logger.isDebugEnabled()) {
+ logger.debug("create index for table {}, sql: {} ", model.getName(), indexSQL.toStringInNewLine());
+ }
+ client.execute(connection, indexSQL.toString());
+ }
}