Python百度文庫爬蟲之pdf文件

Python百度文庫爬蟲之ppt文件

對於文件的所有類型，我都會用一篇文章進行說明，鏈接：

1. Python百度文庫爬蟲之txt文件
2. Python百度文庫爬蟲之doc文件
3. Python百度文庫爬蟲之pdf文件
4. Python百度文庫爬蟲之ppt文件
5. [Python百度文庫爬蟲之xls文件
6. Python百度文件爬蟲終極版

一.網頁分析

說明一下：分析doc文件和pdf文件非常相似，從爬取過程來說基本類似。因此我不會花費太多的語言來說明，直接使用代碼

以此鏈接：https://wenku.baidu.com/view/5fd3a55b3d1ec5da50e2524de518964bcf84d2b6.html?fr=search文件為例說明

from IPython.display import Image
Image("./Images/pdf_0.png",width="600px",height="400px")

Image("./Images/pdf_1.png",width="600px",height="400px")

Image("./Images/pdf_2.png",width="600px",height="400px")

二.數據URL獲取

import requests
import json
import re
import json

session=requests.session()

url=input("請輸入下載的文件URL地址：")

content=session.get(url).content.decode('gbk')
doc_id=re.findall('view/(.*?).html',url)[0]
types=re.findall(r"docType.*?:.*?'(.*?)'",content)[0]
title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

請輸入下載的文件URL地址： https://wenku.baidu.com/view/5fd3a55b3d1ec5da50e2524de518964bcf84d2b6.html?fr=search

title

'網絡爬蟲技術研究'

url_list=re.findall(r'(https.*?0.json.*?)\\x22}',content)
url_list=[addr.replace("\\\\\\/","/") for addr in url_list]

三.程序調試

result=""

for url in set(url_list):
    content=session.get(url).content.decode('gbk')
    
    y=0
    txtlists=re.findall(r'"c":"(.*?)".*?"y":(.*?),',content)
    for item in txtlists:
        # 當item[1]的值與前面不同時，代表要換行了
        if not y==item[1]:
            y=item[1]
            n='\n'
        else:
            n=''
        result+=n
        result+=item[0].encode('utf-8').decode('unicode_escape','ignore')

E:\Anaconda\envs\mypython\lib\site-packages\ipykernel_launcher.py:16: DeprecationWarning: invalid escape sequence '\/'
  app.launch_new_instance()

filename="./Files/"+title+'.txt'

with open(filename,'w',encoding="utf-8") as f:
    f.write(result)
    
f.close()

四.函數編程

1.功能完善

提供下載[‘txt’,‘doc’,‘pdf’,‘ppt’]文件

import requests
import json
import re
import os

session=requests.session()

path="F:\\桌面\\Files"

if not os.path.exists(path):
    os.mkdir(path)

def parse_txt1(code,doc_id):
    
    content_url='https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id='+doc_id

    content=session.get(content_url).content.decode(code)
    md5sum=re.findall('"md5sum":"(.*?)",',content)[0]
    rsign=re.findall('"rsign":"(.*?)"',content)[0]
    pn=re.findall('"totalPageNum":"(.*?)"',content)[0]
    
    content_url='https://wkretype.bdimg.com/retype/text/'+doc_id+'?rn='+pn+'&type=txt'+md5sum+'&rsign='+rsign
    content=json.loads(session.get(content_url).content.decode('gbk'))
    
    result=''

    for item in content:
        for i in item['parags']:
            result+=i['c']
            
    return result

def parse_txt2(content,code,doc_id):
    md5sum=re.findall('"md5sum":"(.*?)",',content)[0]
    rsign=re.findall('"rsign":"(.*?)"',content)[0]
    pn=re.findall('"show_page":"(.*?)"',content)[0]
    
    content_url='https://wkretype.bdimg.com/retype/text/'+doc_id+'?rn='+pn+'&type=txt'+md5sum+'&rsign='+rsign
    content=json.loads(session.get(content_url).content.decode('utf-8'))
    
    result=''

    for item in content:
        for i in item['parags']:
            result+=i['c']
            
    return result

def parse_doc(content):
    
    url_list=re.findall(r'(https.*?0.json.*?)\\x22}',content)
    url_list=[addr.replace("\\\\\\/","/") for addr in url_list]
    
    result=""

    for url in set(url_list):
        content=session.get(url).content.decode('gbk')

        y=0
        txtlists=re.findall(r'"c":"(.*?)".*?"y":(.*?),',content)
        for item in txtlists:
            # 當item[1]的值與前面不同時，代表要換行了
            if not y==item[1]:
                y=item[1]
                n='\n'
            else:
                n=''
            result+=n
            result+=item[0].encode('utf-8').decode('unicode_escape','ignore')
    
    return result

def parse_pdf(content):
    
    url_list=re.findall(r'(https.*?0.json.*?)\\x22}',content)
    url_list=[addr.replace("\\\\\\/","/") for addr in url_list]
    
    result=""

    for url in set(url_list):
        content=session.get(url).content.decode('gbk')

        y=0
        txtlists=re.findall(r'"c":"(.*?)".*?"y":(.*?),',content)
        for item in txtlists:
            # 當item[1]的值與前面不同時，代表要換行了
            if not y==item[1]:
                y=item[1]
                n='\n'
            else:
                n=''
            result+=n
            result+=item[0].encode('utf-8').decode('unicode_escape','ignore')
    
    return result
    

def save_file(title,filename,content):
    
    with open(filename,'w',encoding='utf-8') as f:
        f.write(content)
        print("文件"+title+"保存成功")
    f.close()
    

def main():
    
    print("歡迎來到百度文庫文件下載：")
    print("-----------------------\r\n")
     
    
    while True:
        try:
            print("1.doc \n 2.txt \n 3.ppt \n 4.xls\n 5.ppt\n")
            types=input("請輸入需要下載文件的格式(0退出)：")

            if types=="0":
                break

            if types not in ['txt','doc','pdf']:
                print("抱歉功能尚未開發")
                continue


            url=input("請輸入下載的文庫URL地址：")

            # 網頁內容
            response=session.get(url)

            code=re.findall('charset=(.*?)"',response.text)[0]

            if code.lower()!='utf-8':
                code='gbk'

            content=response.content.decode(code)

            # 文件id
            doc_id=re.findall('view/(.*?).html',url)[0]
            # 文件類型
            #types=re.findall(r"docType.*?:.*?'(.*?)'",content)[0]
            # 文件主題
            #title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

            if types=='txt':
                md5sum=re.findall('"md5sum":"(.*?)",',content)
                if md5sum!=[]:
                    result=parse_txt2(content,code,doc_id)
                    title=re.findall(r'<title>(.*?). ',content)[0]
                    #filename=os.getcwd()+"\\Files\\"+title+'.txt'
                    filename=path+"\\"+title+".txt"
                    save_file(title,filename,result)
                else: 
                    result=parse_txt1(code,doc_id)
                    title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]
                    #filename=os.getcwd()+"\\Files\\"+title+'.txt'
                    filename=path+"\\"+title+".txt"
                    save_file(title,filename,result)
            elif types=='doc':
                title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]
                result=parse_doc(content)
                filename=path+"\\"+title+".doc"
                save_file(title,filename,result)
            elif types=='pdf':
                title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]
                result=parse_pdf(content)
                filename=path+"\\"+title+".txt"
                save_file(title,filename,result)
                
                
        except Exception as e:
            print(e)


if __name__=='__main__':
    main()

歡迎來到百度文庫文件下載：
-----------------------

1.doc 
 2.txt 
 3.ppt 
 4.xls
 5.ppt



請輸入需要下載文件的格式(0退出)： pdf
請輸入下載的文庫URL地址： https://wenku.baidu.com/view/5fd3a55b3d1ec5da50e2524de518964bcf84d2b6.html?fr=search


E:\Anaconda\envs\mypython\lib\site-packages\ipykernel_launcher.py:93: DeprecationWarning: invalid escape sequence '\/'


文件網絡爬蟲技術研究保存成功
1.doc 
 2.txt 
 3.ppt 
 4.xls
 5.ppt



請輸入需要下載文件的格式(0退出)： 0