You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by GitBox <gi...@apache.org> on 2019/09/27 08:13:28 UTC
[GitHub] [incubator-shardingsphere] Zhangkaiqiang commented on issue #3140:
sharding-jdbc batch insert Native memory allocation (mmap) failed to map
Zhangkaiqiang commented on issue #3140: sharding-jdbc batch insert Native memory allocation (mmap) failed to map
URL: https://github.com/apache/incubator-shardingsphere/issues/3140#issuecomment-535840450
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:sharding="http://shardingsphere.apache.org/schema/shardingsphere/sharding"
xmlns:bean="http://www.springframework.org/schema/util"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xmlns:aop="http://www.springframework.org/schema/aop"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/tx
http://www.springframework.org/schema/tx/spring-tx.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd
http://shardingsphere.apache.org/schema/shardingsphere/sharding
http://shardingsphere.apache.org/schema/shardingsphere/sharding/sharding.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd
http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-4.0.xsd
http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop.xsd">
<!-- 自动扫描 -->
<context:component-scan base-package="com.douyin" />
<!-- 引入配置文件 -->
<bean
class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<list>
<value>classpath:jdbc.properties</value>
<value>classpath:redis.properties</value>
<value>classpath:mq.properties</value>
</list>
</property>
</bean>
<bean:properties id="properties">
<prop key="worker.id">123</prop>
</bean:properties>
<bean id="recordIdStandardStrategy" class="com.douyin.table.split.strategy.DateStandardStrategy"/>
<sharding:complex-strategy algorithm-ref="recordIdStandardStrategy" sharding-columns="record_time" id="complexStrategy"/>
<sharding:standard-strategy id="shardingRecordIdStandardStrategy" precise-algorithm-ref="recordIdStandardStrategy" sharding-column="record_id" range-algorithm-ref=""/>
<sharding:standard-strategy id="shardingDateStandardStrategy" range-algorithm-ref="recordIdStandardStrategy" precise-algorithm-ref="recordIdStandardStrategy" sharding-column="create_time"/>
<sharding:key-generator id="orderKeyGenerator" type="SNOWFLAKE" column="create_time" props-ref="properties" />
<sharding:inline-strategy id="orderTableStrategy" sharding-column="create_time" algorithm-expression="pb_import_info_$->{order_id % 2}" />
<sharding:data-source id="shardingDataSource">
<sharding:sharding-rule data-source-names="demo_ds">
<sharding:table-rules>
<sharding:table-rule logic-table="pb_import_info" actual-data-nodes="demo_ds.pb_import_info_$->{0..11}" table-strategy-ref="complexStrategy" key-generator-ref="orderKeyGenerator" />
<!-- <sharding:table-rule logic-table="pb_import_info" actual-data-nodes="demo_ds.pb_import_info_$->{0..11}" table-strategy-ref="shardingRecordIdStandardStrategy" key-generator-ref="orderKeyGenerator" /> -->
</sharding:table-rules>
<sharding:binding-table-rules>
<sharding:binding-table-rule logic-tables="pb_import_info"/>
</sharding:binding-table-rules>
<!-- <sharding:broadcast-table-rules>
<sharding:broadcast-table-rule table="t_address"/>
</sharding:broadcast-table-rules> -->
</sharding:sharding-rule>
<sharding:props>
<prop key="sql.show">true</prop>
</sharding:props>
</sharding:data-source>
<!-- 数据源的配置 -->
<bean id="demo_ds" class="com.alibaba.druid.pool.DruidDataSource"
init-method="init" destroy-method="close">
<property name="driverClassName" value="${jdbc.driver}" />
<property name="url" value="${jdbc.url}" />
<property name="username" value="${jdbc.username}" />
<property name="password" value="${jdbc.password}" />
<!-- data source configuration -->
<property name="initialSize" value="5" /><!-- initial connections -->
<property name="maxActive" value="200" /><!-- MAX connections -->
<!--<property name="maxIdle" value="10" /><!– MAX idle connections –>-->
<property name="minIdle" value="3" /><!-- MIN idle connections -->
<property name="testWhileIdle" value="true" />
<property name="testOnBorrow" value="false" />
<property name="testOnReturn" value="false" />
<property name="validationQuery" value="select 1" />
<property name="timeBetweenEvictionRunsMillis" value="20000" />
<property name="numTestsPerEvictionRun" value="100" />
<property name="filters" value="stat" />
<property name="proxyFilters">
<list>
<ref bean="stat-filter"/>
</list>
</property>
</bean>
<bean id="stat-filter" class="com.alibaba.druid.filter.stat.StatFilter">
<!-- 慢sql时间设置,即执行时间大于200毫秒的都是慢sql -->
<property name="slowSqlMillis" value="200"/>
<property name="logSlowSql" value="true"/>
</bean>
<!--事务相关控制 -->
<bean id="transactionManager"
class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
<property name="dataSource" ref="shardingDataSource" />
</bean>
<!-- spring和MyBatis完美整合，不需要mybatis的配置映射文件 -->
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="shardingDataSource" />
<!-- 自动扫描mapping.xml文件 -->
<property name="mapperLocations" value="classpath:com/douyin/mapping/*.xml"></property>
</bean>
<bean id="sqlSession" class="org.mybatis.spring.SqlSessionTemplate">
<constructor-arg index="0" ref="sqlSessionFactory" />
</bean>
<!-- DAO接口所在包名，Spring会自动查找其下的类 -->
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="com.douyin.dao" />
<property name="sqlSessionFactoryBeanName" value="sqlSessionFactory"></property>
</bean>
<bean id="SpringApplicationContext" class="com.douyin.util.ApplicationContextHelper"></bean>
<!--启动注解 @Transactional -->
<tx:annotation-driven />
<!-- <tx:advice id="txadvice" transaction-manager="transactionManager">
<tx:attributes>
<tx:method name="save*" propagation="REQUIRED"
rollback-for="Exception" />
<tx:method name="update*" propagation="REQUIRED"
rollback-for="Exception" />
<tx:method name="delete*" propagation="REQUIRED"
rollback-for="Exception" />
<tx:method name="insert*" propagation="REQUIRED"
rollback-for="Exception" />
<tx:method name="import*" propagation="REQUIRED"
rollback-for="Exception" />
<tx:method name="*" propagation="REQUIRED" read-only="false" />
</tx:attributes>
</tx:advice>
<aop:config>
<aop:pointcut id="daoMethod" expression="execution(* com.douyin.service.*.*.*(..))" />
<aop:advisor pointcut-ref="daoMethod" advice-ref="txadvice" />
</aop:config> -->
</beans>
@Override
public Collection<String> doSharding(Collection<String> availableTargetNames,
ComplexKeysShardingValue<String> shardingValue) {
// TODO Auto-generated method stub
Map<String,Collection<String>> map = shardingValue.getColumnNameAndShardingValuesMap();
Map<String,Range<String>> rangeMap = shardingValue.getColumnNameAndRangeValuesMap();
Collection<String> result = Lists.newArrayList();
Calendar calendar = Calendar.getInstance();
if(!MapUtils.isEmpty(map)) {
for(String key : map.keySet()) {
if("record_time".equals(key)) {
Collection<String> recordTimeSet = map.get(key);
for(String recordTime : recordTimeSet) {
Date date = DateUtil.parse(recordTime);
result.add(TableConstants.PB_IMPORT_INFO.concat(getTableByRecordTime(date).toString()));
}
}
}
}
if(!MapUtils.isEmpty(rangeMap)) {
for(String key : rangeMap.keySet()) {
if("record_time".equals(key)){
Range<String> range = rangeMap.get(key);
Date startTime = DateUtil.parse(range.lowerEndpoint());
Date endTime = DateUtil.parse(range.upperEndpoint());
if(startTime.equals(endTime)) {
result.add(TableConstants.PB_IMPORT_INFO.concat(getTableByRecordTime(startTime).toString()));
}else {
while(startTime.before(endTime)) {
result.add(TableConstants.PB_IMPORT_INFO.concat(getTableByRecordTime(startTime).toString()));
calendar.setTime(startTime);
calendar.add(Calendar.DAY_OF_WEEK, 1);
startTime = calendar.getTime();
}
}
}
}
}
return result;
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services