You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by "tiandihai (via GitHub)" <gi...@apache.org> on 2023/11/28 01:39:17 UTC
[I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
tiandihai opened a new issue, #29225:
URL: https://github.com/apache/shardingsphere/issues/29225
5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了。
我的分库策略 ds_sharding 是通过SPI方式引入的，5.4.0和5.4.1都正常生效。
但是分表策略 tb_sharding 是通过 type=CLASS_BASED 自定义类的方式引入的，5.4.0正常生效，但是升级到5.4.1之后就失效了，后面我给改成和上面的分库策略一样也通过SPI方式引入，也不生效。不知道是为什么？
另外我有4个库，每个库2000张表左右，升级到5.4.1之后服务启动特别慢(不引人shardingsphere的时候启动速度2分钟左右，引入shardingsphere 5.4.0的时候启动速度5分钟左右，5.4.1启动速度达到20分钟，我把大量的表删除之后启动速度明显提升)，有什么方式能加快扫描表的速度吗？
这个是我的分库策略类：DatabaseShardingAlgorithm.java
这个是我的分表策略类：TableShardingAlgorithm.java
这是我的配置内容如下：
props:
sql-show: true
dataSources:
db-0:
xxxx
db-1:
xxxx
db-2:
xxxx
db-3:
xxxx
sqlFederation:
sqlFederationEnabled: true
executionPlanCache:
initialCapacity: 1000
maximumSize: 1000
rules:
- SQL_PARSER: {}
sqlCommentParseEnabled: true
- SINGLE: {}
tables:
- '*.*'
defaultDataSource: db-0
- SHARDING: {}
tables:
tlog_transnumber_repeat:
actualDataNodes: db-${0..3}.tlog_transnumber_repeat
databaseStrategy:
standard:
shardingColumn: trans_number
shardingAlgorithmName: ds_sharding
tlog_header:
actualDataNodes: db-0.tlog_header
tableStrategy:
standard:
shardingColumn: trans_number
shardingAlgorithmName: tb_sharding
bindingTables:
- b_header,b_items
shardingAlgorithms:
ds_sharding:
type: DATABASE-MOD-SHARDING
props:
database-sharding-ranges: db-0:0-15,db-1:16-31,db-2:32-47,db-3:48-63
tb_sharding:
type: CLASS_BASED
props:
strategy: STANDARD
algorithmClassName: com.yonghui.yh.rme.soc.stream.consumer.sharding.algorithm.TableShardingAlgorithm
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "tiandihai (via GitHub)" <gi...@apache.org>.
tiandihai commented on issue #29225:
URL: https://github.com/apache/shardingsphere/issues/29225#issuecomment-1828924645
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import org.apache.shardingsphere.sharding.api.sharding.hint.HintShardingAlgorithm;
import org.apache.shardingsphere.sharding.api.sharding.hint.HintShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.PreciseShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
/**
* @Description:
* @author wanglei10
* @date 2023-08-02 11:35
*/
@Slf4j
public class DatabaseShardingAlgorithm implements StandardShardingAlgorithm<Comparable<?>>, HintShardingAlgorithm<Comparable<?>> {
private final Map<Integer, String> partitionDatabaseMap = new HashMap<>();
private volatile int partitionSize;
@Override
public void init(Properties properties) {
calculatePartitionRange(properties);
}
private static final String DATABASE_SHARDING_RANGES = "database-sharding-ranges";
public void calculatePartitionRange(final Properties props) {
Preconditions.checkState(props.containsKey(DATABASE_SHARDING_RANGES), "props: database-sharding-ranges cannot be null.");
log.info("database-sharding-ranges配置：{}", props.getProperty(DATABASE_SHARDING_RANGES));
String[] dsConfigs = props.getProperty(DATABASE_SHARDING_RANGES).split(",");
Preconditions.checkState(dsConfigs.length > 0, "props: database-sharding-ranges配置错误,配置实例：db-0:0-15,db-1:16-31,db-2:32-47,db-3:48-63");
for (String s : dsConfigs) {
if (s != null && !s.trim().equals("")) {
String[] s1 = s.trim().split(":");
if (s1.length == 2) {
String dsName = s1[0].trim();
String[] s2 = s1[1].trim().split("-");
int start = Integer.parseInt(s2[0].trim());
int end = Integer.parseInt(s2[1].trim());
for (int i = start; i <= end; i++) {
partitionDatabaseMap.put(i, dsName);
}
} else {
log.warn("props: database-sharding-ranges配置可能出现错误:{}", s);
}
}
}
partitionSize = partitionDatabaseMap.size();
//解析后配置
log.info("database-sharding-ranges解析后配置");
partitionDatabaseMap.entrySet().stream()
.collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.mapping(Map.Entry::getKey, Collectors.toList())))
.forEach((key, value) -> log.info("数据库：{},哈希值:{}-{}", key, value.get(0), value.get(value.size() - 1)));
}
@Override
public Collection<String> doSharding(Collection<String> availableTargetNames, HintShardingValue<Comparable<?>> shardingValue) {
return shardingValue.getValues().stream().map(it -> routeDatabase(availableTargetNames, it)).collect(Collectors.toList());
}
@Override
public final String doSharding(final Collection<String> availableTargetNames, final PreciseShardingValue<Comparable<?>> shardingValue) {
return routeDatabase(availableTargetNames, shardingValue.getValue());
}
private String routeDatabase(Collection<String> availableTargetNames, Comparable<?> value) {
if (value == null) {
throw new RuntimeException("shardingValue不能为空");
}
int slot = (int) (MurmurHash2.hash32(value.toString().getBytes(StandardCharsets.UTF_8), 2773) & 0xFFFFFFFFL % partitionSize);
String dbName = partitionDatabaseMap.get(slot);
if (dbName == null) {
throw new RuntimeException("database-sharding-ranges配置错误:" + value + "对应的配置不存在");
}
if (!availableTargetNames.contains(dbName)) {
throw new RuntimeException("database-sharding-ranges配置错误:" + dbName + "数据库不存在");
}
return dbName;
}
@Override
public final Collection<String> doSharding(final Collection<String> availableTargetNames, final RangeShardingValue<Comparable<?>> shardingValue) {
return availableTargetNames;
}
@Override
public String getType() {
return "DATABASE-MOD-SHARDING";
}
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "tiandihai (via GitHub)" <gi...@apache.org>.
tiandihai commented on issue #29225:
URL: https://github.com/apache/shardingsphere/issues/29225#issuecomment-1828924817
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.shardingsphere.sharding.api.sharding.standard.PreciseShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* @Description:
* @author wanglei10
* @date 2023-08-10 13:55
*/
@Slf4j
public class TableShardingAlgorithm implements StandardShardingAlgorithm<String> {
public static final InheritableThreadLocal<Boolean> NO_SHARDING_HOLDER = new InheritableThreadLocal<>();
/**
* 分片算法的属性配置
*/
private final Properties properties = new Properties();
/**
* 根据精确分片值进行分片操作，返回分片后的目标表名
*
* @param availableTargetNames 所有的目标表名集合
* @param preciseShardingValue 精确分片值对象，包含了逻辑表名、分片键和分片值
* @return 分片后的目标表名
*/
@Override
public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {
return routeTables(availableTargetNames, preciseShardingValue);
}
private String routeTables(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {
if (preciseShardingValue == null) {
throw new RuntimeException("shardingValue不能为空");
}
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
String date = simpleDateFormat.format(new Date());
String actualTableName;
if (BooleanUtils.isTrue(NO_SHARDING_HOLDER.get())) {
actualTableName = preciseShardingValue.getLogicTableName();
}else {
actualTableName = preciseShardingValue.getLogicTableName().concat("_").concat(date);
}
//把真实表名放进来
availableTargetNames.add(actualTableName);
return actualTableName;
}
/**
* 根据范围分片值进行分片操作，返回分片后的目标表名集合
*
* @param availableTargetNames 所有的目标表名集合
* @param rangeShardingValue 范围分片值对象，包含了逻辑表名、分片键和分片范围
* @return 分片后的目标表名集合
*/
@Override
public Collection<String> doSharding(Collection<String> availableTargetNames, RangeShardingValue<String> rangeShardingValue) {
return null;
}
/**
* 获取分片算法的类型
*
* @return 分片算法的类型
*/
@Override
public String getType() {
return "TB_SHARDING_BY_DATE";
}
/**
* 获取分片算法的属性配置
*
* @return 分片算法的属性配置
*/
public Properties getProps() {
return properties;
}
/**
* 初始化分片算法和属性配置
*
* @param properties 分片算法的属性配置
*/
@Override
public void init(Properties properties) {
log.info("初始化按日期分表算法 -> properties:{}", properties.toString());
}
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "RaigorJiang (via GitHub)" <gi...@apache.org>.
RaigorJiang commented on issue #29225:
URL: https://github.com/apache/shardingsphere/issues/29225#issuecomment-1829377102
Welcome to Apache ShardingSphere community, please note only **English** is accepted here.
I will close this issue, please submit a new one if needed.
Thank you!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "tiandihai (via GitHub)" <gi...@apache.org>.
tiandihai commented on issue #29225:
URL: https://github.com/apache/shardingsphere/issues/29225#issuecomment-1828943671
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import org.apache.shardingsphere.sharding.api.sharding.hint.HintShardingAlgorithm;
import org.apache.shardingsphere.sharding.api.sharding.hint.HintShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.PreciseShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
/**
* @Description:
* @author wanglei10
* @date 2023-08-02 11:35
*/
@Slf4j
public class DatabaseShardingAlgorithm implements StandardShardingAlgorithm<Comparable<?>>, HintShardingAlgorithm<Comparable<?>> {
private final Map<Integer, String> partitionDatabaseMap = new HashMap<>();
private volatile int partitionSize;
@Override
public void init(Properties properties) {
calculatePartitionRange(properties);
}
private static final String DATABASE_SHARDING_RANGES = "database-sharding-ranges";
public void calculatePartitionRange(final Properties props) {
Preconditions.checkState(props.containsKey(DATABASE_SHARDING_RANGES), "props: database-sharding-ranges cannot be null.");
log.info("database-sharding-ranges配置：{}", props.getProperty(DATABASE_SHARDING_RANGES));
String[] dsConfigs = props.getProperty(DATABASE_SHARDING_RANGES).split(",");
Preconditions.checkState(dsConfigs.length > 0, "props: database-sharding-ranges配置错误,配置实例：db-0:0-15,db-1:16-31,db-2:32-47,db-3:48-63");
for (String s : dsConfigs) {
if (s != null && !s.trim().equals("")) {
String[] s1 = s.trim().split(":");
if (s1.length == 2) {
String dsName = s1[0].trim();
String[] s2 = s1[1].trim().split("-");
int start = Integer.parseInt(s2[0].trim());
int end = Integer.parseInt(s2[1].trim());
for (int i = start; i <= end; i++) {
partitionDatabaseMap.put(i, dsName);
}
} else {
log.warn("props: database-sharding-ranges配置可能出现错误:{}", s);
}
}
}
partitionSize = partitionDatabaseMap.size();
//解析后配置
log.info("database-sharding-ranges解析后配置");
partitionDatabaseMap.entrySet().stream()
.collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.mapping(Map.Entry::getKey, Collectors.toList())))
.forEach((key, value) -> log.info("数据库：{},哈希值:{}-{}", key, value.get(0), value.get(value.size() - 1)));
}
@Override
public Collection<String> doSharding(Collection<String> availableTargetNames, HintShardingValue<Comparable<?>> shardingValue) {
return shardingValue.getValues().stream().map(it -> routeDatabase(availableTargetNames, it)).collect(Collectors.toList());
}
@Override
public final String doSharding(final Collection<String> availableTargetNames, final PreciseShardingValue<Comparable<?>> shardingValue) {
return routeDatabase(availableTargetNames, shardingValue.getValue());
}
private String routeDatabase(Collection<String> availableTargetNames, Comparable<?> value) {
if (value == null) {
throw new RuntimeException("shardingValue不能为空");
}
int slot = (int) (MurmurHash2.hash32(value.toString().getBytes(StandardCharsets.UTF_8), 2773) & 0xFFFFFFFFL % partitionSize);
String dbName = partitionDatabaseMap.get(slot);
if (dbName == null) {
throw new RuntimeException("database-sharding-ranges配置错误:" + value + "对应的配置不存在");
}
if (!availableTargetNames.contains(dbName)) {
throw new RuntimeException("database-sharding-ranges配置错误:" + dbName + "数据库不存在");
}
return dbName;
}
@Override
public final Collection<String> doSharding(final Collection<String> availableTargetNames, final RangeShardingValue<Comparable<?>> shardingValue) {
return availableTargetNames;
}
@Override
public String getType() {
return "DATABASE-MOD-SHARDING";
}
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "RaigorJiang (via GitHub)" <gi...@apache.org>.
RaigorJiang closed issue #29225: 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了
URL: https://github.com/apache/shardingsphere/issues/29225
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "tiandihai (via GitHub)" <gi...@apache.org>.
tiandihai commented on issue #29225:
URL: https://github.com/apache/shardingsphere/issues/29225#issuecomment-1828925413
`import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.shardingsphere.sharding.api.sharding.standard.PreciseShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* @Description:
* @author wanglei10
* @date 2023-08-10 13:55
*/
@Slf4j
public class TableShardingAlgorithm implements StandardShardingAlgorithm<String> {
public static final InheritableThreadLocal<Boolean> NO_SHARDING_HOLDER = new InheritableThreadLocal<>();
/**
* 分片算法的属性配置
*/
private final Properties properties = new Properties();
/**
* 根据精确分片值进行分片操作，返回分片后的目标表名
*
* @param availableTargetNames 所有的目标表名集合
* @param preciseShardingValue 精确分片值对象，包含了逻辑表名、分片键和分片值
* @return 分片后的目标表名
*/
@Override
public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {
return routeTables(availableTargetNames, preciseShardingValue);
}
private String routeTables(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {
if (preciseShardingValue == null) {
throw new RuntimeException("shardingValue不能为空");
}
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
String date = simpleDateFormat.format(new Date());
String actualTableName;
if (BooleanUtils.isTrue(NO_SHARDING_HOLDER.get())) {
actualTableName = preciseShardingValue.getLogicTableName();
}else {
actualTableName = preciseShardingValue.getLogicTableName().concat("_").concat(date);
}
//把真实表名放进来
availableTargetNames.add(actualTableName);
return actualTableName;
}
/**
* 根据范围分片值进行分片操作，返回分片后的目标表名集合
*
* @param availableTargetNames 所有的目标表名集合
* @param rangeShardingValue 范围分片值对象，包含了逻辑表名、分片键和分片范围
* @return 分片后的目标表名集合
*/
@Override
public Collection<String> doSharding(Collection<String> availableTargetNames, RangeShardingValue<String> rangeShardingValue) {
return null;
}
/**
* 获取分片算法的类型
*
* @return 分片算法的类型
*/
@Override
public String getType() {
return "TB_SHARDING_BY_DATE";
}
/**
* 获取分片算法的属性配置
*
* @return 分片算法的属性配置
*/
public Properties getProps() {
return properties;
}
/**
* 初始化分片算法和属性配置
*
* @param properties 分片算法的属性配置
*/
@Override
public void init(Properties properties) {
log.info("初始化按日期分表算法 -> properties:{}", properties.toString());
}
}`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [I] 5.4.0升级到5.4.1之后，CLASS_BASED 方式自定义类的tableStrategy(分表策略)失效了 [shardingsphere]
Posted by "tiandihai (via GitHub)" <gi...@apache.org>.
tiandihai commented on issue #29225:
URL: https://github.com/apache/shardingsphere/issues/29225#issuecomment-1828943183
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.shardingsphere.sharding.api.sharding.standard.PreciseShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* @Description:
* @author wanglei10
* @date 2023-08-10 13:55
*/
@Slf4j
public class TableShardingAlgorithm implements StandardShardingAlgorithm<String> {
public static final InheritableThreadLocal<Boolean> NO_SHARDING_HOLDER = new InheritableThreadLocal<>();
/**
* 分片算法的属性配置
*/
private final Properties properties = new Properties();
/**
* 根据精确分片值进行分片操作，返回分片后的目标表名
*
* @param availableTargetNames 所有的目标表名集合
* @param preciseShardingValue 精确分片值对象，包含了逻辑表名、分片键和分片值
* @return 分片后的目标表名
*/
@Override
public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {
return routeTables(availableTargetNames, preciseShardingValue);
}
private String routeTables(Collection<String> availableTargetNames, PreciseShardingValue<String> preciseShardingValue) {
if (preciseShardingValue == null) {
throw new RuntimeException("shardingValue不能为空");
}
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
String date = simpleDateFormat.format(new Date());
String actualTableName;
if (BooleanUtils.isTrue(NO_SHARDING_HOLDER.get())) {
actualTableName = preciseShardingValue.getLogicTableName();
}else {
actualTableName = preciseShardingValue.getLogicTableName().concat("_").concat(date);
}
//把真实表名放进来
availableTargetNames.add(actualTableName);
return actualTableName;
}
/**
* 根据范围分片值进行分片操作，返回分片后的目标表名集合
*
* @param availableTargetNames 所有的目标表名集合
* @param rangeShardingValue 范围分片值对象，包含了逻辑表名、分片键和分片范围
* @return 分片后的目标表名集合
*/
@Override
public Collection<String> doSharding(Collection<String> availableTargetNames, RangeShardingValue<String> rangeShardingValue) {
return null;
}
/**
* 获取分片算法的类型
*
* @return 分片算法的类型
*/
@Override
public String getType() {
return "TB_SHARDING_BY_DATE";
}
/**
* 获取分片算法的属性配置
*
* @return 分片算法的属性配置
*/
public Properties getProps() {
return properties;
}
/**
* 初始化分片算法和属性配置
*
* @param properties 分片算法的属性配置
*/
@Override
public void init(Properties properties) {
log.info("初始化按日期分表算法 -> properties:{}", properties.toString());
}
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org