go http爬虫

1 

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	resp,err := http.Get("http://yeves.cn")
	if err != nil{
		fmt.Print("http get err",err)
		return
	}

	body,err := ioutil.ReadAll(resp.Body)
	if err != nil{
		fmt.Print("http get err",err)
		return
	}

	fmt.Print(string(body))
}




2
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
)

func main() {
	url := "http://yeves.cn"
	download(url)
}

func download(url string) {
	client := &http.Client{}
	req,_ := http.NewRequest("GET",url,nil)

	req.Header.Set("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
	resp,err := client.Do(req)
	if err != nil{
		fmt.Print("http get err",err)
		return
	}

	defer resp.Body.Close()

	body,err := ioutil.ReadAll(resp.Body)
	if err != nil{
		fmt.Print("read error ",err)
		return
	}
	fmt.Print(string(body))
}


3

package main

import (
	"fmt"
	"github.com/jackdanger/collectlinks"
	"net/http"
)

func main() {
	url := "http://www.baidu.com/"
	download(url)
}

func download(url string) {
	client := &http.Client{}
	req,_ := http.NewRequest("GET",url,nil)

	req.Header.Set("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
	resp,err := client.Do(req)
	if err != nil{
		fmt.Print("http get err",err)
		return
	}

	defer resp.Body.Close()

	links := collectlinks.All(resp.Body)
	for _,link := range links{
		fmt.Print("parse url",link+"\n")
	}
}