A good example is a User-Agent switcher which changes User-Agent on every request:

 

Configuration | Colly http://go-colly.org/docs/introduction/configuration/

 

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

func RandomString() string {
	b := make([]byte, rand.Intn(10)+10)
	for i := range b {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	return string(b)
}

c := colly.NewCollector()

c.OnRequest(func(r *colly.Request) {
	r.Headers.Set("User-Agent", RandomString())
})

 

posted @ 2018-08-23 17:15  papering  阅读(176)  评论(0编辑  收藏  举报