Python學習之實現簡單的高並發爬蟲爬取網頁

import gevent,time
from  urllib import request
#urllib的io操作gevent不會識別，不會自動切換，以下方法解決
from gevent import monkey
monkey.patch_all() #把當前程序的所有Io操作給我單獨的做上標記

def f(url):
    print('GET: %s' % url)
    resp = request.urlopen(url)
    data = resp.read()
    # f = open("url.html","wb")
    # f.write(data)
    # f.close()
    print('%d bytes received from %s.' % (len(data), url))

#串行
urls = [
    'https://www.python.org/',
    'https://www.nginx.org/',
    'https://github.com/',
]
time_start = time.time()
for url in urls:
    f(url)
print("同步cost",time.time()-time_start)

#協程並行
async_time_start = time.time()
gevent.joinall([
    gevent.spawn(f, 'https://www.python.org/'),
    gevent.spawn(f, 'https://www.yahoo.com/'),
    gevent.spawn(f, 'https://github.com/'),
])
print("異步cost",time.time()-async_time_start)