go語言爬蟲goquery和grequests的使用

/*下載工具*/ package main import ( "fmt" //go語言版本的jquery "github.com/PuerkitoBio/goquery" "os" "sync" "strings" //go語言版本的request "github.com/levigross/grequests" "time" "strconv" ) var wg sync.WaitGroup func main() { now := time.Now() initalUrls := []string{"http://www.zngirls.com/girl/18071/album/", } for _, url := range initalUrls { doc, err := goquery.NewDocument(url) if err != nil { fmt.Errorf("下載錯誤:%#v", err) os.Exit(-1) } doc.Find(".igalleryli_link").Each(func(i int, s *goquery.Selection) { src, exists := s.Find("img").Attr("src") fmt.Printf("開始下載影集圖片:%v\n", src) if (exists) { wg.Add(1) go func(src string) { defer wg.Done() //下載圖片 //tryTimes := map[int]int n := 0 s := strings.Replace(src, "cover/", "", 1) ss := strings.Split(s, "/") fm := strings.Join(ss[:len(ss) - 1], "/") sf0 := fm + "/%d.jpg" sfn := fm + "/%03d.jpg" for { //持續下載 s := "" if n == 0 { s = fmt.Sprintf(sf0, n) } else { s = fmt.Sprintf(sfn, n) } fmt.Printf("准備下載: %v\n", s) res, _ := grequests.Get(s, &grequests.RequestOptions{ //結構體可以對指定的類型給值，而不一定都賦值 Headers:map[string]string{ "Referer":"http://www.zngirls.com", "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"}}) //條件需要修改，如果沒有圖片，返回的是盜鏈，圖片4kb if res.StatusCode != 200 { fmt.Printf("下載失敗，退出影集下載:%s\n", src) break } //圖片可能是該網站，返回的盜鏈圖片（4kb左右) length := res.Header.Get("Content-Length") slen,_ := strconv.Atoi(length) if slen < 4100{ fmt.Printf("下載內容失敗,退出影集下載:%s\n", src) break } index := strings.Index(s, "gallery") if index == -1 { fmt.Errorf("無效地址，找不到gallery關鍵詞，解析失敗:%s\n", src) return } ss2 := strings.Split(string(s[index:]), "/") dirname := strings.Join(ss2[:len(ss2) - 1], "/") if _, err := os.Stat(dirname); err != nil { fmt.Printf("創建下載文件夾:%s\n", dirname) os.MkdirAll(dirname, 0666) } filename := strings.Join(ss2, "/") res.DownloadToFile(filename) fmt.Printf("成功下載圖片到:%s\n", filename) n++ } }(src) } }) } wg.Wait() //4M的帶寬下載，需要16m36s，總大小202M,10個文件夾,560個文件 fmt.Printf("下載任務完成，耗時:%#v\n", time.Now().Sub(now)) }