GO-小爬虫

小爬虫百度

package main

import (
	"net/http"
	"fmt"
)

func main() {
	response,err := http.Get("https://www.cnblogs.com/")
	fmt.Println(err)
	html := make([]byte,4*1024)
	var text string
	for {
		n,err := response.Body.Read(html)
		if n==0 {
			fmt.Println(err)
			break
		}
		text += string(html[:n])
	}

	fmt.Println(html)
	fmt.Println(text)
}

小爬虫头条

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://profile.zjurl.cn/api/feed_backflow/profile_share/v1/?category=profile_all&visited_uid=4492956276&stream_api_version=82&request_source=1&offset=0&user_id=4492956276&media_id=4492956276&appId=1286&appType=mobile_detail_web&isAndroid=true&isIOS=false&isMobile=true&cookie_enabled=true&screen_width=360&screen_height=640&browser_language=zh-CN&browser_platform=MacIntel&browser_name=firefox&browser_version=86.0.4240.198&browser_online=true&timezone_name=Asia%2FShanghai&_signature=_02B4Z6wo00f01.B2THAAAIBDMzwCVjJYeoPwY0jAAKOz5JXVy3OFm4k-aHdT6AiigXlVpy45kcV6KjskZEAS7XQy7WA0lPFcufCvvuLciQUTc-wbErkTeTZb2qEIL2E2sAB-4w9P1bHT5syxeb", nil)
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Set("authority", "profile.zjurl.cn")
	req.Header.Set("pragma", "no-cache")
	req.Header.Set("cache-control", "no-cache")
	req.Header.Set("accept", "text/javascript, text/html, application/xml, text/xml, */*")
	req.Header.Set("x-requested-with", "XMLHttpRequest")
	req.Header.Set("user-agent", "Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Mobile Safari/537.36")
	req.Header.Set("content-type", "application/x-www-form-urlencoded")
	req.Header.Set("sec-fetch-site", "same-origin")
	req.Header.Set("sec-fetch-mode", "cors")
	req.Header.Set("sec-fetch-dest", "empty")
	req.Header.Set("referer", "https://profile.zjurl.cn/rogue/ugc/profile/?user_id=4492956276&media_id=4492956276&request_source=1")
	req.Header.Set("accept-language", "zh-CN,zh;q=0.9")
	req.Header.Set("cookie", "gftoken=MTk5MzAzMTMzMHwxNjA1NjA1NjE1ODV8fDAGBgYGBgY; s_v_web_id=verify_khls4mmz_lIx8FRXf_1oLb_4pfA_BFQp_MXI45BTMTftn; MONITOR_WEB_ID=41bd0cdb-d37e-4837-984d-8cea0a602167; tt_scid=HVkHoh30TyEopsYRtYQmPzLKkhG8-1Q5Ajn.phCrimoCdtgURQkAI3XzepXGkP45d4d6")
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	bodyText, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", bodyText)
}