MongoDB查詢重復記錄並保存到文件csv

 

客戶1w用戶記錄，發現里面有小部分重復數據

需要查出，比對哪些信息不同

https://docs.mongodb.org/manual/reference/operator/aggregation/#aggregation-pipeline-operator-reference

https://docs.mongodb.org/manual/reference/operator/aggregation/group/#pipe._S_group

https://docs.mongodb.org/manual/reference/operator/aggregation/addToSet/#grp._S_addToSet

 

var keys = "";
db.users.find().limit(1).forEach(function(u){
    for(var p in u)
    {
        keys += p + ",";
    }    
});

keys = keys.trimRight(",");
print(keys); //輸出csv列名
db.users.aggregate([{$group: { _id: "$prid", values: {$addToSet: "$$CURRENT"}, total: {$sum: 1}}}, {$match: {total: {$gt: 1}}}]).forEach(function(g){
    g.values.forEach(function(v){
        var line = "";
        for(var key in v)
        {
            line += v[key] + ",";
        }
        line = line.trimRight(",");
        print(line); //輸出重復數據
    })
})

保存上述代碼比如到D:\mongojs\aggregate.js

運行

mongo yourdb aggregate.js > repeated.records.csv