mysql歷史數據自動歸檔

數據庫跑一段時間后，因為查詢性能、磁盤容量，運維管理等方面的原因，需要將在線數據挪到歷史庫(不同的服務器）。如我們的在線訂單只留3個月數據，3個月以前的就需要到歷史庫查了。

自動歸檔常見的方式有pt-archiver，但我還是覺得自己寫存儲過程更靠譜。。。

思路：

在線庫實例打開federated支持，創建數據庫dborder(業務庫), linkhis(歸檔用);
歷史庫創建歷史表dborderhis.myorder_tab_his;
在linkhis庫下創建federated表linkhis.myorder_tab_his，指向dborderhis.myorder_tab_his；
在linkhis庫下創建日志表archive_log，存儲過程proc_archive，proc_archive_pkg，並通過JOB調度proc_archive_pkg；
在線庫的從庫需要忽略linkhis的復制：replicate-ignore-db=linkhis，否則從庫也會往這個歷史庫重復同步數據。

日志記錄表archive_log

create table archive_log
(
id bigint auto_increment PRIMARY key,
tab_name varchar(40),
archive_date_begin datetime,
archive_date_end datetime,
create_time datetime default CURRENT_TIMESTAMP(),
status int(1),
insert_rows bigint(11),
delete_rows bigint(11),
remark varchar(1000)
)

 

存儲過程：proc_archive

CREATE PROCEDURE proc_archive(in i_table_source varchar(40), 
in i_table_target varchar(40), 
in i_fieldname varchar(40), 
in i_keepdays int,
in i_archdays int,
in i_other_cond varchar(500))
begin
/*
入參：
i_table_source:原表,含dbname
i_table_target:federated表
i_fieldname:時間字段
i_keepdays:保留天數
i_archdays:每次歸檔多少天數據
i_other_cond:數據額外條件(如status in (2,3)不能歸檔，需要保留),無額外條件則輸入'1=1'
歸檔日志表archive_log.status字段含義：
0:成功, 1:現有數據在保留天數內, 2:目標表含有待歸檔時間范圍的數據, 
3:插入數據和刪除數據記錄數不同, 4:SQL執行異常，具體錯誤見remark
注意：
有額外條件時，如果歷史數據被修改，從不符合歸檔條件變成符合歸檔條件，
因歷史表中歸檔時間段內已經有之前歸檔的數據(@v_his_num_before>0)，程序會退出，需手動處理
*/
declare EXIT HANDLER for SQLWARNING,NOT FOUND,SQLEXCEPTION 
begin 
GET DIAGNOSTICS CONDITION 1 @p1=RETURNED_SQLSTATE,@p2= MESSAGE_TEXT;
ROLLBACK;
insert into archive_log(tab_name,archive_date_begin,archive_date_end,status,insert_rows,delete_rows,remark)
values(i_table_source,@v_arch_begin,@v_arch_end,4,@v_his_num_after,@v_del_num,concat('error ',@p1,' - ',@p2));
end;
/* 獲取在線表的最小日期 */
set @mystmt = concat("select str_to_date(date_format(min(",i_fieldname,"),'%Y%m%d'),'%Y%m%d') into @v_arch_begin from ",i_table_source,' where ',i_other_cond);
prepare stmt from @mystmt;
execute stmt;
deallocate prepare stmt;

set @v_arch_end = date_add(@v_arch_begin,interval i_archdays day);

set @mystmt = concat("select count(*) into @v_his_num_before from ",i_table_target," where ",i_fieldname," >= ? and ",i_fieldname," < ?");
prepare stmt from @mystmt;
execute stmt using @v_arch_begin,@v_arch_end;
deallocate prepare stmt;
/* 如果在線表的數據低於keepday范圍，退出 */
if timestampdiff(day,@v_arch_begin,now()) <= i_keepdays then
insert into archive_log(tab_name,archive_date_begin,archive_date_end,status,insert_rows,delete_rows,remark)
values(i_table_source,@v_arch_begin,@v_arch_end,1,0,0,concat('error, all data in keey days, min ',i_fieldname,': ',@v_arch_begin));
end if;
/* 如果歷史表所在的日期區間有數據，退出（需要手動排查原因） */
if @v_his_num_before <> 0 then
insert into archive_log(tab_name,archive_date_begin,archive_date_end,status,insert_rows,delete_rows,remark)
values(i_table_source,@v_arch_begin,@v_arch_end,2,0,0,concat('error, data exists,row num:',@v_his_num_before));
end if;

if (timestampdiff(day,@v_arch_begin,now()) > i_keepdays and @v_his_num_before = 0) then 
set @mystmt = concat("insert into ",i_table_target," select * from ",i_table_source," where ",i_fieldname," >= ? and ",i_fieldname," < ? and ",i_other_cond);
prepare stmt from @mystmt;
execute stmt using @v_arch_begin,@v_arch_end;
deallocate prepare stmt;
/* 因為federated引擎不支持事務，數據insert后再select下記錄數，與下面的delete記錄數對比，相同則提交delete操作 */
set @mystmt = concat("select count(*) into @v_his_num_after from ",i_table_target," where ",i_fieldname," >= ? and ",i_fieldname," < ?");
prepare stmt from @mystmt;
execute stmt using @v_arch_begin,@v_arch_end;
deallocate prepare stmt;

start transaction;

set @mystmt = concat("delete from ",i_table_source," where ",i_fieldname," >= ? and ",i_fieldname," < ? and ",i_other_cond);
prepare stmt from @mystmt;
execute stmt using @v_arch_begin,@v_arch_end;
set @v_del_num = row_count();
deallocate prepare stmt;

if @v_del_num = @v_his_num_after then
commit;
insert into archive_log(tab_name,archive_date_begin,archive_date_end,status,insert_rows,delete_rows,remark)
values(i_table_source,@v_arch_begin,@v_arch_end,0,@v_his_num_after,@v_del_num,'success');
else
rollback;
insert into archive_log(tab_name,archive_date_begin,archive_date_end,status,insert_rows,delete_rows,remark)
values(i_table_source,@v_arch_begin,@v_arch_end,3,@v_his_num_after,@v_del_num,'rollback, inserted rows num not equal to deleted rows num');
end if;
end if;

end;
 

 

存儲過程proc_archive_pkg

 

CREATE PROCEDURE `proc_archive_pkg`()
begin
call proc_archive(
                  'dborder.myorder_tab', -- tabel source
                  'myorder_tab_his',     -- table target
                  'create_time',         -- time field name
                  120,                   -- i_keepdays
                  1,                     -- i_archdays
                  '1=1'                  -- i_other_cond
                  );
 
end

 

歸檔日志表記錄