python3 爬蟲（urllib+beautifulsoup）beautifulsoup自動檢測編碼錯誤

版本：Python3.x

運行系統：win7

編輯器：pycharm

爬取頁面：攜程的一個頁面（韓國首爾6日5晚半自助游·直飛+滑雪場或南怡島+樂天世界+1天自由活動-【攜程旅游】）

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup
def getComment(url):
    try:
        html = urlopen(url)
    except HTTPError as e:
        return None    #網頁在服務器上不存在，若服務器不存在直接返回None
    try:
        soup = BeautifulSoup(html.read(),"lxml")
        comment = soup.body.find("ul",{"class":"detail_comment_list"}).find("li")
    except ArithmeticError as e:
        return None
    return comment
comment = getComment("http://vacations.ctrip.com/grouptravel/p11504202s32.html#ctm_ref=va_hom_s32_prd_p1_l2_2_img")
if comment == None:
    print("comment could not be found")
else:
    comment 1 = comment.get_text()
    print(comment 1)
但是會輸出亂碼
解決方案：
（1）使用第三方庫requests+beautifulsoup，requests對編碼有比較好的處理能力
（2）直接編碼檢測，發現網頁標注是UTF-8寫的，但其實是gbk編碼的編碼

from  urllib.request import urlopen
import chardet
a = urlopen('http://vacations.ctrip.com/grouptravel/p11504202s32.html#ctm_ref=va_hom_s32_prd_p1_l2_2_img').read()
b = chardet.detect(a)
print(b)
#{'encoding': 'GB2312', 'confidence': 0.99}
利用beautifulsoup對網頁編碼確認

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup
def getComment(url):
    try:
        html = urlopen(url)
    except HTTPError as e:
        return None    #網頁在服務器上不存在，若服務器不存在直接返回None
    try:
        soup = BeautifulSoup(html.read(),"lxml",from_encoding='gbk')
        comment = soup.body.find("ul",{"class":"detail_comment_list"}).find("li")
    except ArithmeticError as e:
        return None
    return comment
comment = getComment("http://vacations.ctrip.com/grouptravel/p11504202s32.html#ctm_ref=va_hom_s32_prd_p1_l2_2_img")
if comment == None:
    print("commmnt could not be found")
else:
    comment1 = comment.get_text()
    print(comment1)