Mysql Json性能测试数据构建

因项目需要尝试使用mysql json解决问题：

主表：

create table `json_test_main` (
   `id` bigint  not null auto_increment comment '主键',
   `content` char(1024)  not null comment '内容',
   `json_str` json  not null comment '内容',
   primary key (id)
) comment='json_test_main';


关联表：

create table `json_test_ref` (
   `id` bigint  not null auto_increment comment '主键',
   `ref_id` bigint  not null  comment 'ref键',
   `asset_type` varchar(512)  not null comment '资产类型',
   `asset_value` varchar(512)  not null comment '资产值',
   primary key (id)
) comment='json_test_main';

ref_id 为主表中的id 

上述两个表是问题简化后的数据结构，可描述为 main 表的一条记录根据 json字段 asset_type asset_value 建立倒排索引到ref表。
数据构建程序生成main表数据，同时对于main表中的每一个记录生成倒排索引表记录 。通常是1对多的关系。
可以控制main表中 json中的类型个数 和 每一个类型设置的value list 的长度确定倒排长度。


上代码：

package jizg.study.maven.hello;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

import org.apache.commons.collections.map.HashedMap;
import org.springframework.jdbc.core.JdbcTemplate;

/**
 * Created by lkpnotice on 2017/5/25.
 */
public class DataGenerator {

    /* 五种类型 */

    String[] typesArray = new String[]{"type1","type2","type3","type4","type5"};
    int  assetValRange = 1000;
    int  assetValLimit = 10;
    String content = "[e-mail]";

    public static final int mainFlushLimit = 100;
    public static final int refFlushLimit = 200;


    public DataGenerator(){
    }

    public DataGenerator(JdbcTemplate jdbc){
        this.jdbcTemplate = jdbc;
    }


    public static void main(String[] args){
        DataGenerator gen = new DataGenerator();
        Map<String,List<String>> unit =  gen.genUnit();
        gen.genUnitMain(1,unit);
        gen.genUnitRef(1,unit);
        System.out.println(gen.mainMap);
        System.out.println(gen.refMap);
    }

    /**
     *
     * @param idBase
     */
    public void genLoopCase1(long idBase,long ammount){
        for (long cc = 0 ; cc < ammount ; cc++){
            long idNow = idBase + 1  +cc;
            genLoopOnce(idNow);
        }

        flushLoopLeft();
    }



    /**/
    public void genLoopOnce(long id){
        /*gen new unit */
        Map<String,List<String>> unit = genUnit();
        /*transform to main and ref unit object*/
        genUnitMain(id,unit);
        genUnitRef(id,unit);
        if (mainCount >= mainFlushLimit){/*main table db op and flush ,empty cache and counter*/
            handleDbMainBatchInsert();
        }

        if (refCount >= refFlushLimit){/*maintable db op and flush ,empty cache and counter*/
            handlerDbRefBatchInsert();
        }
    }


    /* left batch commit*/
    public void flushLoopLeft(){
        if (mainCount >= 0){/*main table db op and flush ,empty cache and counter*/
            handleDbMainBatchInsert();

        }

        if (refCount >= 0){/*maintable db op and flush ,empty cache and counter*/
            handlerDbRefBatchInsert();
        }
    }

    public void clearMain(){
        mainMap.clear();
    }

    public void clearRef(){
        refMap.clear();
    }


    JdbcTemplate jdbcTemplate;
    String mainInsertSql ="insert into json_test_main_0001(id,content,json_str)  values (?,?,?)";
    public void handleDbMainBatchInsert(){
        List<Object[]>  batchArgs = new ArrayList<Object[]>();
        for (MainTable main: mainMap.values()
             ) {
            Long id = main.getId();
            String content = main.getContent();
            String jsonStr = main.getJsonStr();
            batchArgs.add(new Object[]{id,content,jsonStr});
        }
        int[] returns = jdbcTemplate.batchUpdate(mainInsertSql, batchArgs);
        clearMain();
    }

    String refInsertSql ="insert into json_test_ref_0001(ref_id,asset_type,asset_value) values (?,?,?)";
    public void handlerDbRefBatchInsert(){
        List<Object[]>  batchArgs = new ArrayList<Object[]>();
        for (List<RefTable> refs:refMap.values()
             ) {
            for (RefTable ref: refs
                 ) {
                //ref.getId();
                Long refId = ref.getRefId();
                String assetType = ref.getAssetType();
                String assetValue = ref.getAssetVal();
                batchArgs.add(new Object[]{refId,assetType,assetValue});
            }
        }

        int[] returns = jdbcTemplate.batchUpdate(refInsertSql, batchArgs);
        clearRef();
    }



    /*RefTable*/
    Map<Long,List<RefTable>> refMap = new HashMap();
    public int refCount=0;
    public void genUnitRef(long id,Map<String,List<String>> unit){
        List<RefTable> refList = new ArrayList();
        Iterator<Entry<String,List<String>>> iter =  unit.entrySet().iterator();
        while(iter.hasNext()){
            Entry<String,List<String>> en = iter.next();
            String key = en.getKey();
            List<String> valList = en.getValue();
            for (String str:valList
                 ) {
                RefTable ref = new RefTable();
                ref.setRefId(id);
                ref.setAssetType(key);
                ref.setAssetVal(str);
                refList.add(ref);
            }
        }
        refCount += refList.size();
        refMap.put(id,refList);
    }




    /*mainTable*/
    Map<Long,MainTable>  mainMap = new HashedMap();
    public int mainCount = 0;
    public void genUnitMain(long id,Map<String,List<String>> unit){
        String jsonStr =  JSON.toJSONString(unit);
        MainTable main = new MainTable();
        main.setId(id);
        main.setContent(content);
        main.setJsonStr(jsonStr);
        mainCount += 1;
        mainMap.put(id,main);
    }



    public Map<String,List<String>> genUnit(){
        /*gen unit type */
        int len = typesArray.length;

        /*gen how many types of this id*/
        int  count =  (int)Math.round(Math.max((Math.random() * len) ,1));

        Map<String,List<String>> unit = new HashedMap();
        /*gen each type of this id*/
        for(int i = 0;i < count ; i++){
            int index =  (int)Math.round(Math.max((Math.random() * len) -1,0));
            String assetType = typesArray[index];
            List<String> tmpType = unit.get(assetType);
            if (null == tmpType){
                /*gen how many val of this type*/
                int cnt = (int)Math.round(Math.max((Math.random() * assetValLimit),1));
                List<String> valList  =  new ArrayList();
                 /*gen each val of this type*/
                for (int j = 0; j< cnt ; j++){
                    long assetVal = Math.round(Math.random() * assetValRange);
                    valList.add(new Long(assetVal).toString());
                }

                unit.put(assetType,valList);
            }else{
                i --;
            }

        }
        return unit;

    }



    class MainTable {
        long id;
        String content;
        String jsonStr;
        JSONObject json;

        public long getId() {
            return id;
        }

        public void setId(long id) {
            this.id = id;
        }

        public String getContent() {
            return content;
        }

        public void setContent(String content) {
            this.content = content;
        }

        public String getJsonStr() {
            return jsonStr;
        }

        public void setJsonStr(String jsonStr) {
            this.jsonStr = jsonStr;
        }

        public JSONObject getJson() {
            return json;
        }

        public void setJson(JSONObject json) {
            this.json = json;
        }

        @Override
        public String toString() {
            return id  + " : " + content + " : " + jsonStr;
        }
    }



    class  RefTable {
        long id;
        long refId;
        String assetType;
        String assetVal;

        public long getId() {
            return id;
        }

        public void setId(long id) {
            this.id = id;
        }

        public long getRefId() {
            return refId;
        }

        public void setRefId(long refId) {
            this.refId = refId;
        }

        public String getAssetType() {
            return assetType;
        }

        public void setAssetType(String assetType) {
            this.assetType = assetType;
        }

        public String getAssetVal() {
            return assetVal;
        }

        public void setAssetVal(String assetVal) {
            this.assetVal = assetVal;
        }

        @Override
        public String toString() {
            return id +" : " + refId + " : " + assetType  + " : " + assetVal;
        }
    }


}