Sharding-JDBC+MyBatis分库分表

　　随着项目功能越来越多业务越来越复杂，数据库存储的数据逐渐庞大，当mysql单表存储数据过千万的时候，对该表的操作变得缓慢，这时候就需要通过分库分表对数据库优化。

 

水平分库：是把同一个表的数据按一定规则拆到不同的数据库中，每个库可以放在不同的服务器上。

• 解决了单库大数据，高并发的性能瓶颈
• 提高了系统的稳定性及可用性

水平分表：是在同一个数据库内，把同一个表的数据按一定规则拆到多个表中。

• 优化单一表数据量过大而产生的性能问题
• 避免IO争抢并减少锁表的几率

 

　　本文采用 Sharding-JDBC 分库分表，它是一个优秀的开源框架，使用客户端直连数据库，以 jar 包形式提供服务，无需额外部署和依赖，可理解为增强版的 JDBC 驱动，完全兼容 JDBC 和各种 ORM 框架。

• 适用于任何基于 JDBC 的 ORM 框架，如：JPA, Hibernate, Mybatis, Spring JDBC Template 或直接使用 JDBC。
• 支持任何第三方的数据库连接池，如：DBCP, C3P0, BoneCP, Druid, HikariCP 等。
• 支持任意实现 JDBC 规范的数据库，目前支持 MySQL，Oracle，SQLServer，PostgreSQL 以及任何遵循 SQL92 标准的数据库。

一、建库建表

CREATE DATABASE ds_0;
USE ds_0;

DROP TABLE IF EXISTS `user_0`;
CREATE TABLE `user_0` (
  `id` bigint(64) NOT NULL,
  `city` varchar(20) NOT NULL,
  `name` varchar(20) NOT NULL,
  `sex` tinyint(4) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

DROP TABLE IF EXISTS `user_1`;
CREATE TABLE `user_1` (
  `id` bigint(64) NOT NULL,
  `city` varchar(20) NOT NULL,
  `name` varchar(20) NOT NULL,
  `sex` tinyint(4) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE DATABASE ds_1;
USE ds_1;

DROP TABLE IF EXISTS `user_0`;
CREATE TABLE `user_0` (
  `id` bigint(64) NOT NULL,
  `city` varchar(20) NOT NULL,
  `name` varchar(20) NOT NULL,
  `sex` tinyint(4) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

DROP TABLE IF EXISTS `user_1`;
CREATE TABLE `user_1` (
  `id` bigint(64) NOT NULL,
  `city` varchar(20) NOT NULL,
  `name` varchar(20) NOT NULL,
  `sex` tinyint(4) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

 

二、创建SpringBoot项目，引入依赖

<dependency>
     <groupId>org.mybatis.spring.boot</groupId>
     <artifactId>mybatis-spring-boot-starter</artifactId>
     <version>1.3.1</version>
</dependency>
<dependency>
    <groupId>io.shardingjdbc</groupId>
    <artifactId>sharding-jdbc-core</artifactId>
    <version>2.0.3</version>
</dependency>
<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>druid</artifactId>
    <version>1.1.3</version>
</dependency>
<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.44</version>
</dependency>

三、创建mapper文件、类、实体

UserMapper.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="law.mapper.UserMapper">

    <resultMap id="baseResultMap" type="law.entity.User">
        <result column="id" property="id"/>
        <result column="city" property="city"/>
        <result column="name" property="name"/>
        <result column="sex" property="sex"/>
    </resultMap>

    <insert id="addUser" parameterType="law.entity.User">
        INSERT INTO user (id, city, name, sex)
        VALUES (#{id}, #{city}, #{name}, #{sex})
    </insert>

    <select id="list" resultMap="baseResultMap" parameterType="java.util.ArrayList">
        SELECT * FROM user
        <where>
            id in (
            <foreach collection="list" item="id" index="index" separator=",">
                #{id}
            </foreach>
            )
        </where>
    </select>

    <select id="query" resultMap="baseResultMap">
        SELECT * FROM user order by id asc
    </select>

</mapper>

package law.mapper;

import law.entity.User;
import org.springframework.stereotype.Repository;
import java.util.List;

@Repository
public interface UserMapper {

    void addUser(User user);

    List<User> list(List<Long> ids);

    List<User> query();
}

package law.entity;

import lombok.Data;
import java.io.Serializable;

@Data
public class User implements Serializable {

    private static final long serialVersionUID = -1205226416664488559L;

    private Long id;

    private String city;

    private String name;

    private Long sex;
}

四、加载数据库链接、创建分库分表策略

package law.configs;

import com.alibaba.druid.pool.DruidDataSource;
import io.shardingjdbc.core.api.config.ShardingRuleConfiguration;
import io.shardingjdbc.core.api.config.TableRuleConfiguration;
import io.shardingjdbc.core.api.config.strategy.StandardShardingStrategyConfiguration;
import io.shardingjdbc.core.jdbc.core.datasource.ShardingDataSource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;

import javax.sql.DataSource;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;

@Configuration
@MapperScan(basePackages = "law.mapper", sqlSessionTemplateRef = "testSqlSessionTemplate")
public class DataSourceConfig {

    /**
     * 配置数据源
     */
    @Bean(name = "shardingDataSource")
    DataSource getShardingDataSource() throws SQLException {
        ShardingRuleConfiguration shardingRuleConfig = new ShardingRuleConfiguration();
        shardingRuleConfig.getTableRuleConfigs().add(userTableRuleConfiguration());
        shardingRuleConfig.getTableRuleConfigs().add(caipiaoTableRuleConfiguration());
        return new ShardingDataSource(shardingRuleConfig.build(createDataSourceMap()));
    }

    /**
     * 设置单表的分库分表策略
     * 虚拟表与实际表的映射关系
     */
    @Bean
    TableRuleConfiguration userTableRuleConfiguration() {
        TableRuleConfiguration configuration = new TableRuleConfiguration();
        configuration.setLogicTable("user");
        configuration.setActualDataNodes("ds_${0..1}.user_${0..1}");
        configuration.setDatabaseShardingStrategyConfig(new StandardShardingStrategyConfiguration("id", DatabaseShardingAlgorithm.class.getName()));
        configuration.setTableShardingStrategyConfig(new StandardShardingStrategyConfiguration("sex", DatabaseShardingAlgorithm.class.getName()));
        return configuration;
    }

    /**
     * 设置不分库分表的映射关系
     */
    @Bean
    TableRuleConfiguration caipiaoTableRuleConfiguration() {
        TableRuleConfiguration configuration = new TableRuleConfiguration();
        configuration.setLogicTable("caipiao");
        configuration.setActualDataNodes("wms.caipiao");
        return configuration;
    }

    /**
     * 手动配置事务管理器
     */
    @Bean
    public DataSourceTransactionManager transactitonManager(DataSource shardingDataSource) {
        return new DataSourceTransactionManager(shardingDataSource);
    }

    @Bean
    @Primary
    public SqlSessionFactory sqlSessionFactory(DataSource shardingDataSource) throws Exception {
        SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
        bean.setDataSource(shardingDataSource);
        bean.setMapperLocations(new PathMatchingResourcePatternResolver().getResources("classpath:law/mapper/*.xml"));
        return bean.getObject();
    }

    @Bean
    @Primary
    public SqlSessionTemplate testSqlSessionTemplate(SqlSessionFactory sqlSessionFactory) throws Exception {
        return new SqlSessionTemplate(sqlSessionFactory);
    }

    /**
     * 加载数据库
     */
    private Map<String, DataSource> createDataSourceMap() {
        Map<String, DataSource> result = new HashMap<>();
        result.put("ds_0", createDataSource("ds_0"));
        result.put("ds_1", createDataSource("ds_1"));
        result.put("wms", createDataSource("wms"));
        return result;
    }

    private DataSource createDataSource(final String dataSourceName) {
        DruidDataSource result = new DruidDataSource();
        result.setDriverClassName(com.mysql.jdbc.Driver.class.getName());
        result.setUrl(String.format("jdbc:mysql://localhost:3306/%s", dataSourceName) + "?useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&zeroDateTimeBehavior=convertToNull&useSSL=false");
        result.setUsername("root");
        result.setPassword("123456");

        return result;
    }
}

package law.configs;

import io.shardingjdbc.core.api.algorithm.sharding.PreciseShardingValue;
import io.shardingjdbc.core.api.algorithm.sharding.standard.PreciseShardingAlgorithm;
import java.util.Collection;

/**
*分库分表算法，分为2个库每个库2个表
*以id字段分库，用id值除2取余，分配到对应库
*以sex字段分表，用sex值除2取余，分配到对应表
*/
public class DatabaseShardingAlgorithm implements PreciseShardingAlgorithm<Long> {
    @Override
    public String doSharding(Collection<String> collection, PreciseShardingValue<Long> preciseShardingValue) {
        for (String each : collection) {
            if (each.endsWith(Long.parseLong(preciseShardingValue.getValue().toString()) % 2 + "")) {
                return each;
            }
        }
        throw new IllegalArgumentException();
    }
}

五、启动测试

package law.controller;

import com.google.common.collect.Lists;
import law.entity.Lottery;
import law.entity.User;
import law.mapper.LotteryMapper;
import law.mapper.UserMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.List;

@RestController
public class UserController {

    @Autowired
    private UserMapper userMapper;
    @Autowired
    private LotteryMapper lotteryMapper;

    @Transactional
    @RequestMapping("/addUsers")
    public Object addUsers() {
        for (long i = 1; i < 10; i++) {
            User user = new User();
            user.setId(i);
            user.setCity("北京");
            user.setName("张" + i);
            user.setSex((long) 0);
            userMapper.addUser(user);
        }
//        //测试事务
//        if (1 == 1) {
//            throw new RuntimeException();
//        }
        for (long i = 11; i < 20; i++) {
            User user = new User();
            user.setId(i);
            user.setCity("成都");
            user.setName("刘" + i);
            user.setSex((long) 1);
            userMapper.addUser(user);
        }
        return "success";
    }

　　//测试单表
    @RequestMapping("/addLotteries")
    public Object addLotteries() {
        for (int i = 1; i < 3; i++) {
            Lottery lottery = new Lottery();
            lottery.setCpNum(i);
            lottery.setCpValue("票" + i);
            lotteryMapper.addLottery(lottery);
        }
        return "success";
    }

    @RequestMapping("/selectByIds")
    public Object selectByIds() {
        List<Long> ids = Lists.newArrayList();
        ids.add(1L);
        ids.add(3L);
        return userMapper.list(ids);
    }

    @RequestMapping("/selectSort")
    public Object selectSort() {
        return userMapper.query();
    }
}

package law;

import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.transaction.annotation.EnableTransactionManagement;

@MapperScan(value = {"law.mapper"})
@EnableTransactionManagement(proxyTargetClass = true)
@SpringBootApplication
public class Demo {
    public static void main(String[] args) {
        SpringApplication.run(Demo.class, args);
    }
}