開發背景
多個大表數據均值3-5億,故使用mysql 分庫分表策略 水平拆分成小表
工程引入依賴
<!--shardingsphere 分庫分表-->
<dependency>
<groupId>org.apache.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
<version>${sharding-sphere.version}</version>
</dependency>
<dependency>
<groupId>org.apache.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-namespace</artifactId>
<version>${sharding-sphere.version}</version>
</dependency>
引入nacos配置
記一次不知原因的問題: 分庫分表的配置 tables 配置三個可用,兩個可用,四個不可用,五個可用。 即為了可用性,配置一張虛擬表到五張表配置!!!!!!!!!!

# 數據源 spring: datasource: type: com.alibaba.druid.pool.DruidDataSource druid: driver-class-name: com.mysql.cj.jdbc.Driver username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true stat-view-servlet: enabled: true url-pattern: /druid/* #login-username: admin #login-password: admin filter: stat: enabled: true log-slow-sql: true slow-sql-millis: 10000 merge-sql: false wall: config: multi-statement-allow: true #sharding-jdbc shardingsphere: datasource: names: history,master,slave1,slave2,mswkhis # 主庫 master: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 從庫1 slave1: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 從庫2 slave2: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/${MYSQL-DB:ms_wk}?characterEncoding=utf8&allowPublicKeyRetrieval=true&allowPublicKeyRetrieval=true&zeroDateTimeBehavior=convertToNull&useSSL=false&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8&allowMultiQueries=true username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 歷史數據庫 history: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/ms_history?serverTimezone=UTC&useSSL=false&useUnicode=true&characterEncoding=UTF-8 username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) # 分庫分表數據庫 mswkhis: driver-class-name: com.mysql.cj.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://${MYSQL-HOST:msdp-mysql}:${MYSQL-PORT:9306}/ms_wk_his?serverTimezone=UTC&useSSL=false&useUnicode=true&characterEncoding=UTF-8 username: ${MYSQL-USER:ms_wk} password: ENC(cYIdeHxqoBk7BXyBhLliz5YqQLlD5kPs) props: sql: show: true # 分片配置 sharding: # 分片讀寫分離配置 master-slave-rules: # 默認主從 ds_master_slave: master-data-source-name: master slave-data-source-names: slave1,slave2 # 分片數據源讀寫分離配置(歷史數據源) ds_history: master-data-source-name: history slave-data-source-names: history # 分片數據源讀寫分離配置(分庫分表數據源) ds_ms_wk_his: master-data-source-name: mswkhis slave-data-source-names: mswkhis # 未配置分片規則的表將通過默認數據源定位 default-data-source-name: ds_master_slave binding-tables: wk_sync_log broadcast-tables: t_address tables: wk_sync_log: actual-data-nodes: ds_history.wk_sync_log_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: create_time wk_atdmac_record: actual-data-nodes: ds_history.wk_atdmac_record_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: show_time wk_worker_attendance: actual-data-nodes: ds_ms_wk_his.wk_worker_attendance_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: attend_time wk_worker_attend_month: actual-data-nodes: ds_ms_wk_his.wk_worker_attend_month_$->{2020..2021}$->{['01','02','03','04','05','06','07','08','09','10','11','12']}$->{['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']} key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: attend_date t_sharding_bak: actual-data-nodes: ds_ms_wk_his.t_sharding_bak_20210101 key-generator: column: jid type: UUID table-strategy: standard: precise-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayShardingAlgorithm range-algorithm-class-name: com.iyysoft.cloud.msdp.wk.config.shardingsphere.DayRangeShardingAlgorithm sharding-column: attend_date
設置分片規則

package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import com.google.common.collect.Range; import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.LinkedHashSet; /** * 日期范圍分片 * * @author lvlinguang * @date 2020-08-10 19:09 */ public class DayRangeShardingAlgorithm implements RangeShardingAlgorithm<String> { /** * 設置分片 * * @param collection * @param rangeShardingValue * @return */ @Override public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<String> rangeShardingValue) { Collection<String> result = new LinkedHashSet<>(); DateFormat sdf = new SimpleDateFormat("yyyyMMdd"); //日期 Range<String> ranges = rangeShardingValue.getValueRange(); Date startTime = dateFormat(ranges.lowerEndpoint()); Date endTime = dateFormat(ranges.upperEndpoint()); Calendar cal = Calendar.getInstance(); while (startTime.getTime() <= endTime.getTime()) { String value = sdf.format(startTime); for (String each : collection) { if (each.endsWith(value)) { result.add(each); break; } } cal.setTime(startTime); cal.add(Calendar.DAY_OF_YEAR, 1); startTime = cal.getTime(); } if (result.size() == 0) { result = collection; } return result; } /** * 日期轉換 * * @param date * @return */ public Date dateFormat(String date) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); try { return sdf.parse(date); } catch (ParseException e) { return null; } } }

package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import lombok.SneakyThrows; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue; import java.text.ParseException; import java.util.Collection; /** * 日期精確分片 * * @author lvlinguang * @date 2020-08-10 19:11 */ public class DayShardingAlgorithm implements PreciseShardingAlgorithm<String> { /** * 設置分片 * * @param tableNames 數據表 * @param shardingValue 分片列信息 * @return */ @SneakyThrows @Override public String doSharding(Collection<String> tableNames, PreciseShardingValue<String> shardingValue) { String tableName = shardingValue.getLogicTableName(); String key = getDate(shardingValue.getValue(), 8); return tableName.concat("_").concat(key); } /** * 得到日期數字 * * @param date 字符串日期 * @param len 長度 * @return 202008 * @throws ParseException */ public String getDate(String date, int len) throws ParseException { String number = date.replaceAll("\\D", ""); return number.substring(0, len); } }

package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import com.google.common.collect.Range; import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.LinkedHashSet; /** * 月范圍分片 * * @author lvlinguang * @date 2020-08-10 19:09 */ public class MonthRangeShardingAlgorithm implements RangeShardingAlgorithm<String> { /** * 設置分片 * * @param collection * @param rangeShardingValue * @return */ @Override public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<String> rangeShardingValue) { Collection<String> result = new LinkedHashSet<>(); DateFormat sdf = new SimpleDateFormat("yyyyMM"); //日期 Range<String> ranges = rangeShardingValue.getValueRange(); Date startTime = dateFormat(ranges.lowerEndpoint()); Date endTime = dateFormat(ranges.upperEndpoint()); Calendar cal = Calendar.getInstance(); while (startTime.getTime() <= endTime.getTime()) { String value = sdf.format(startTime); for (String each : collection) { if (each.endsWith(value)) { result.add(each); break; } } cal.setTime(startTime); cal.add(Calendar.MONTH, 1); startTime = cal.getTime(); } if (result.size() == 0) { result = collection; } return result; } /** * 日期轉換 * * @param date * @return */ public Date dateFormat(String date) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); try { return sdf.parse(date); } catch (ParseException e) { return null; } } }

package com.iyysoft.cloud.msdp.wk.config.shardingsphere; import lombok.SneakyThrows; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm; import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue; import java.text.ParseException; import java.util.Collection; /** * 月精確分片 * * @author lvlinguang * @date 2020-08-10 19:11 */ public class MonthShardingAlgorithm implements PreciseShardingAlgorithm<String> { /** * 設置分片 * * @param tableNames 數據表 * @param shardingValue 分片列信息 * @return */ @SneakyThrows @Override public String doSharding(Collection<String> tableNames, PreciseShardingValue<String> shardingValue) { String tableName = shardingValue.getLogicTableName(); String key = getDate(shardingValue.getValue(), 6); return tableName.concat("_").concat(key); } /** * 得到日期數字 * * @param date 字符串日期 * @param len 長度 * @return 202008 * @throws ParseException */ public String getDate(String date, int len) throws ParseException { String number = date.replaceAll("\\D", ""); return number.substring(0, len); } }
大表數據歸檔
-- 創建定時任務事件
create event his_wk_worker_attendance_joint_event
ON SCHEDULE EVERY 1 DAY STARTS DATE_ADD(DATE_ADD(CURDATE(), INTERVAL 1 DAY), INTERVAL 1 HOUR)
on completion preserve enable
do call his_wk_worker_attendance_joint();
-- 查看定時任務
SELECT event_name,event_definition,interval_value,interval_field,status FROM information_schema.EVENTS;
-- 啟停已經創建好的event(事件)
alter event his_wk_worker_attendance_joint_event on completion preserve enable; -- 開啟定時任務
alter event his_wk_worker_attendance_joint_event on completion preserve disable;-- 關閉定時任務
-- 開啟事件調度器
SET GLOBAL event_scheduler = ON;
-- 關閉事件調度器
SET GLOBAL event_scheduler = OFF;
-- 查看事件調度器狀態
SHOW VARIABLES LIKE 'event_scheduler';

CREATE DEFINER=`croot`@`%` PROCEDURE `his_wk_worker_attendance_joint`() BEGIN insert into ms_wk.wk_worker_attendance_joint_his select * from wk_worker_attendance_joint where create_date < date_sub(now(),interval 5 day); commit; delete from ms_wk.wk_worker_attendance_joint where create_date < date_sub(now(),interval 5 day); commit; END
大表數據遷移

CREATE DEFINER=`croot`@`%` PROCEDURE `sub_wk_worker_attendance`() BEGIN declare x int; declare y int; declare z int; set x=2020; set y=1; set z=1; while x<=2022 do while y<=12 do while z<=31 do set @sql_create_table_gpstrail = concat( 'CREATE TABLE IF NOT EXISTS wk_worker_attendance_',lPAD(x,4,'0'),lPAD(y,2,'0'),lPAD(z,2,'0'), "(`jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '在jtg平台的數據id ', `tenant_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '所屬租戶', `create_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '創建人', `create_date` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '創建時間', `update_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '更新人', `update_date` datetime NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新時間', `has_use` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '是否啟用', `has_del` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT '0' COMMENT '是否刪除', `project_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '項目id', `team_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '班組id', `worker_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '工人id', `worker_name` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '工人姓名', `attend_time` datetime NOT NULL COMMENT '考勤時間', `direction` char(2) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '考勤方向 考勤方向字典表: \r\n1入場 \r\n0出場 ', `attend_type` char(3) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '001' COMMENT '通行方式 參考通行方式字典表 :\r\n001 人臉識別\r\n002 虹膜識別\r\n003 指紋識別\r\n004 掌紋識別\r\n005 身份證識別\r\n006 實名卡\r\n007 異常清退(適用於人員沒有通過閘機系統出工地而導致人員狀態不一致的情況)\r\n008 一鍵開閘(需要與閘機交互)\r\n009 應急通道(不需要與閘機交互)\r\n010 二維碼識別\r\n011 其他方式\r\n012系統自動簽出', `img_url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '考勤照片 期望是絕對路徑', `lng` decimal(18, 15) NULL DEFAULT NULL COMMENT '經度 WGS84經度', `lat` decimal(18, 15) NULL DEFAULT NULL COMMENT '緯度 WGS84緯度', `channel` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '通道', `attend_source_type` int(3) UNSIGNED NOT NULL DEFAULT 0 COMMENT '考勤數據來源: 0未定義 , 10個人app考勤 , 11班組長app考勤 , 12管工端考勤機模式考勤 , 20 考勤機考勤 \r\n定義: 1X APP端考勤 , 2X 硬件設備考勤', `atdmac_jid` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '考勤設備jid', PRIMARY KEY (`jid`) USING BTREE, UNIQUE INDEX `attend_time`(`attend_time`, `project_jid`, `worker_jid`, `has_del`) USING BTREE, INDEX `team_jid`(`team_jid`) USING BTREE, INDEX `project_jid`(`project_jid`, `worker_jid`) USING BTREE ) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '工人考勤數據原始記錄' ROW_FORMAT = Dynamic"); PREPARE sql_create_table_gpstrail FROM @sql_create_table_gpstrail; EXECUTE sql_create_table_gpstrail; set z=z+1; end while; set y=y+1; set z=1; end while; set x= x +1; set y=1; set z=1; end while; END

CREATE DEFINER=`croot`@`%` PROCEDURE `insert_wk_worker_attendance`() BEGIN declare x int; declare y int; declare z int; set x=2020; set y=1; set z=1; while x<=2022 do while y<=12 do while z<=31 do set @sql_create_table_gpstrail = concat('insert into ms_wk_his.wk_worker_attendance_',lPAD(x,4,'0'),lPAD(y,2,'0'),lPAD(z,2,'0'), " select * from ms_wk.wk_worker_attendance_bak where attend_time BETWEEN '",lPAD(x,4,'0'),"-",lPAD(y,2,'0'),"-",lPAD(z,2,'0')," 00:00:00' AND '",lPAD(x,4,'0'),"-",lPAD(y,2,'0'),"-",lPAD(z,2,'0')," 23:59:59'"); PREPARE sql_create_table_gpstrail FROM @sql_create_table_gpstrail; EXECUTE sql_create_table_gpstrail; commit; set z=z+1; end while; set y=y+1; set z=1; end while; set x= x +1; set y=1; set z=1; end while; END

CREATE DEFINER=`croot`@`%` PROCEDURE `subDBtest`() BEGIN declare x int; declare y int; declare z int; set x=2020; set y=1; set z=1; while x<=2022 do while y<=12 do while z<=31 do select concat(lPAD(x,4,'0'),lPAD(y,2,'0'),lPAD(z,2,'0')); set z=z+1; end while; set y=y+1; set z=1; end while; set x= x +1; set y=1; set z=1; end while; END
數據遷移問題 走索引 + 存儲過程腳本 會快一點
分庫分表引發的問題
分庫分表不支持子查詢
分庫分表不支持mybatis-plus分頁查詢(自帶子查詢)
分庫分表不帶上分表字段,即會查詢所有分表
// 棄用mybatis-plus自帶分頁,手動分頁參考文獻
https://blog.csdn.net/lq2418c/article/details/120039274?utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7Edefault-3.no_search_link&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7Edefault-3.no_search_link
https://blog.csdn.net/qq_44086060/article/details/116273587

package com.iyysoft.msdp.basic.bean.util; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; /** * @author cjq * @date 2021-09-27 * mybatis-plus 分頁幫助類 */ @Component public class PageHelperUtils { /** * 分頁方法 * * @param currentPage 頁數 * @param pageSize 分頁大小 * @param list 分頁對象 * @return */ public static Page getPages(Integer currentPage, Integer pageSize, List list) { Page page = new Page(); int size = list.size(); if(size == 0){ return page.setCurrent(currentPage).setSize(pageSize).setTotal(list.size()).setRecords(list); } if (pageSize > size) { pageSize = size; } //求出最大頁數,防止currentPage越界 int maxPage = size % pageSize == 0 ? size / pageSize : size / pageSize + 1; if (currentPage > maxPage) { currentPage = maxPage; } //當前頁第一條數據下標 int curIds = currentPage > 1 ? (currentPage - 1) * pageSize : 0; List pageList = new ArrayList<>(); //將當前頁的數據放進pageList for (int i = 0; i < pageSize && curIds + i < size; i++) { pageList.add(list.get(curIds + i)); } page.setCurrent(currentPage).setSize(pageSize).setTotal(list.size()).setRecords(pageList); return page; } }