golang colly下载图片

package main

import (
    "bytes"
    "fmt"
    "github.com/gocolly/colly"
    "io/ioutil"
    "log"
    "strconv"
    "strings"
    "sync"
)

func down(i int, group *sync.WaitGroup) {
    //func down(i int) {
    var c = colly.NewCollector()
    c.OnRequest(func(r *colly.Request) {
        r.Headers.Set("cookie", "__yjs_duid=1_54772224670dec235da1fe2f38979cd31642487126183; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1642487108,1642487119,1642487123,1642487131; zkhanecookieclassrecord=%2C54%2C53%2C; yjs_js_security_passport=7f28c9b52fda683cc710865e91dcec0027759517_1642495580_js; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1642496144")
        r.Headers.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36")

    })
    c.OnError(func(_ *colly.Response, err error) {
        log.Println("Something went wrong:", err)
    })
    c.OnHTML("#main .slist ul li img[src]", func(e *colly.HTMLElement) {
        fmt.Println(e.Attr("src"))
        url := "https://pic.netbian.com" + e.Attr("src")

        var c = colly.NewCollector()
        c.OnRequest(func(r *colly.Request) {
            r.Headers.Set("cookie", "__yjs_duid=1_54772224670dec235da1fe2f38979cd31642487126183; Hm_lvt_526caf4e20c21f06a4e9209712d6a20e=1642487108,1642487119,1642487123,1642487131; zkhanecookieclassrecord=%2C54%2C53%2C; Hm_lpvt_526caf4e20c21f06a4e9209712d6a20e=1642494377; yjs_js_security_passport=7f28c9b52fda683cc710865e91dcec0027759517_1642495580_js")
            r.Headers.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36")

        })
        c.OnResponse(func(r *colly.Response) {
            reader := bytes.NewReader(r.Body)
            body, _ := ioutil.ReadAll(reader)
            //读取图片内容
            index := strings.LastIndex(url, "/")
            s := url[index+1:]
            err := ioutil.WriteFile("./img/"+s, body, 0755)
            fmt.Println(url, err)
        })
        c.Visit(url)

    })
    c.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting", r.URL.String())
    })
    c.OnScraped(func(r *colly.Response) {
        group.Done()
    })
    c.Visit("https://pic.netbian.com/4kmeinv/index_" + strconv.Itoa(i) + ".html")

}

func main() {
    group := sync.WaitGroup{}
    for i := 2; i < 100; i++ {
        group.Add(1)
        go down(i, &group)
    }
    group.Wait()
    fmt.Println("下载完整")
}

 

posted @ 2022-01-18 17:36  酷酷的城池  阅读(545)  评论(0编辑  收藏  举报