Python序列刪除重復數據

## 對於列表來說，若不保持原有順序，可以直接轉換為set刪除重復數據

nums = [1,2,32,2,2,4,3,2,3,42]
nums = list(set(nums))
print(nums)
# [32, 1, 2, 3, 4, 42]  # 刪除了重復數據，但是原有順序也改變了

 

## 刪除數據並保持原有順序

def dedupe(items, key=None):
    """
    items: 哈希或者不可哈希的序列
    key: 若items為不可哈希的序列(dict等)則需要指定一個函數
    """
    seen = set()
    for item in items:
        val = item if key is None else key(item)
        if val not in seen:
            yield item
            seen.add(val)

nums = [1,2,32,2,2,4,3,2,3,42]
print(list(dedupe(nums)))
# [1, 2, 32, 4, 3, 42]

students = [
    {"name": "Stanley", "score": 88},
    {"name": "Lily", "score": 92},
    {"name": "Bob", "score": 91},
    {"name": "Well", "score": 80},
    {"name": "Bob", "score": 90},
    {"name": "Peter", "score": 80}
]
deduped_students = list(dedupe(students, key=lambda s: s['name']))
print(deduped_students)
"""
[{'name': 'Stanley', 'score': 88},
{'name': 'Lily', 'score': 92},
{'name': 'Bob', 'score': 91},
{'name': 'Well', 'score': 80},
{'name': 'Peter', 'score': 80}]   # 刪除了相同姓名的元素
"""
# 刪除姓名和分數都相同的元素
deduped_students = list(dedupe(students, key=lambda s: (s['name'], s['score'])))

 

參考資料：
　　Python Cookbook, 3rd edition, by David Beazley and Brian K. Jones (O’Reilly).