golang colly下载图片
package main import ( "bytes" "fmt" "github.com/gocolly/colly" "io/ioutil" "log" "strconv" "strings" "sync" ) func down(i int, group *sync.WaitGroup) { //func down(i int) { var c = colly.NewCollector() c.OnRequest(func(r *colly.Request) { r.Headers.Set("cookie", "__yjs_duid=1_54772224670dec235da1fe2f38979cd31642487126183; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1642487108,1642487119,1642487123,1642487131; zkhanecookieclassrecord=%2C54%2C53%2C; yjs_js_security_passport=7f28c9b52fda683cc710865e91dcec0027759517_1642495580_js; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1642496144") r.Headers.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36") }) c.OnError(func(_ *colly.Response, err error) { log.Println("Something went wrong:", err) }) c.OnHTML("#main .slist ul li img[src]", func(e *colly.HTMLElement) { fmt.Println(e.Attr("src")) url := "https://pic.netbian.com" + e.Attr("src") var c = colly.NewCollector() c.OnRequest(func(r *colly.Request) { r.Headers.Set("cookie", "__yjs_duid=1_54772224670dec235da1fe2f38979cd31642487126183; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1642487108,1642487119,1642487123,1642487131; zkhanecookieclassrecord=%2C54%2C53%2C; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1642494377; yjs_js_security_passport=7f28c9b52fda683cc710865e91dcec0027759517_1642495580_js") r.Headers.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36") }) c.OnResponse(func(r *colly.Response) { reader := bytes.NewReader(r.Body) body, _ := ioutil.ReadAll(reader) //读取图片内容 index := strings.LastIndex(url, "/") s := url[index+1:] err := ioutil.WriteFile("./img/"+s, body, 0755) fmt.Println(url, err) }) c.Visit(url) }) c.OnRequest(func(r *colly.Request) { fmt.Println("Visiting", r.URL.String()) }) c.OnScraped(func(r *colly.Response) { group.Done() }) c.Visit("https://pic.netbian.com/4kmeinv/index_" + strconv.Itoa(i) + ".html") } func main() { group := sync.WaitGroup{} for i := 2; i < 100; i++ { group.Add(1) go down(i, &group) } group.Wait() fmt.Println("下载完整") }